import os import re import time import logging import requests from bs4 import BeautifulSoup import psycopg2 from psycopg2.extras import execute_values from datetime import datetime # -- CONFIG ------------------------------------------------------------------- PROXY = 'socks5h://127.0.0.1:1090' PROXIES = {'http': PROXY, 'https': PROXY} DB_HOST = '100.75.240.87' DB_PORT = 5432 DB_NAME = 'trabajando_cl' DB_USER = 'pgadmin' DB_PASS = 'J5BVlq65JvedWxZVrcY96OQX' BASE_URL = 'https://www.trabajando.cl' LIST_URL = 'https://www.trabajando.cl/trabajo?page={page}' UA = ( 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' 'AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/124.0.0.0 Safari/537.36' ) DELAY = 0.8 MAX_PAGES = 20 NEW_STREAK = 3 logging.basicConfig( level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S', ) log = logging.getLogger(__name__) def get_session(): sess = requests.Session() sess.proxies.update(PROXIES) sess.headers.update({ 'User-Agent': UA, 'Accept-Language': 'es-CL,es;q=0.9', 'Accept': ( 'text/html,application/xhtml+xml,' 'application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' ), 'Accept-Encoding': 'gzip, deflate, br', 'Connection': 'keep-alive', 'Upgrade-Insecure-Requests': '1', 'Cache-Control': 'max-age=0', }) return sess def create_table(conn): ddl_create = """ CREATE TABLE IF NOT EXISTS jobs ( job_id TEXT PRIMARY KEY, titulo TEXT, empresa TEXT, ubicacion TEXT, url TEXT UNIQUE, fecha_pub TEXT, scraped_at TIMESTAMP DEFAULT NOW() ); """ columns = [ ('titulo', 'TEXT'), ('empresa', 'TEXT'), ('ubicacion', 'TEXT'), ('url', 'TEXT'), ('fecha_pub', 'TEXT'), ('scraped_at', 'TIMESTAMP DEFAULT NOW()'), ] with conn.cursor() as cur: cur.execute(ddl_create) for col, typedef in columns: try: cur.execute( f"ALTER TABLE jobs ADD COLUMN IF NOT EXISTS {col} {typedef};" ) except Exception as exc: log.debug("ALTER TABLE ignorado para columna %s: %s", col, exc) conn.commit() log.info("Tabla jobs verificada/creada correctamente.") def upsert(conn, rows): if not rows: return 0, 0 job_ids = [r['job_id'] for r in rows] with conn.cursor() as cur: cur.execute( "SELECT job_id FROM jobs WHERE job_id = ANY(%s);", (job_ids,) ) existing = {row[0] for row in cur.fetchall()} tuples = [ ( r['job_id'], r.get('titulo'), r.get('empresa'), r.get('ubicacion'), r.get('url'), r.get('fecha_pub'), ) for r in rows ] sql = """ INSERT INTO jobs (job_id, titulo, empresa, ubicacion, url, fecha_pub) VALUES %s ON CONFLICT (job_id) DO UPDATE SET titulo = EXCLUDED.titulo, empresa = EXCLUDED.empresa, ubicacion = EXCLUDED.ubicacion, url = EXCLUDED.url, fecha_pub = EXCLUDED.fecha_pub, scraped_at = NOW(); """ execute_values(cur, sql, tuples) conn.commit() inserted = sum(1 for r in rows if r['job_id'] not in existing) updated = len(rows) - inserted return inserted, updated def scrape_page(sess, url): backoffs = [2, 5, 15] resp = None for attempt, wait in enumerate(backoffs, 1): try: resp = sess.get(url, timeout=30) if resp.status_code in (403, 429): log.warning( "HTTP %s en %s (intento %s) - sleep 30s y reintento extra", resp.status_code, url, attempt ) time.sleep(30) resp = sess.get(url, timeout=30) if resp.status_code in (403, 429): log.warning( "HTTP %s persiste tras reintento extra - devolviendo []", resp.status_code ) return [] resp.raise_for_status() break except requests.RequestException as exc: log.warning( "Error intento %s/%s en %s: %s", attempt, len(backoffs), url, exc ) if attempt < len(backoffs): time.sleep(wait) else: log.error("Todos los intentos fallaron para %s", url) return [] if resp is None or not resp.text.strip(): log.warning("Respuesta vacia para %s", url) return [] soup = BeautifulSoup(resp.text, 'lxml') results = [] cards = ( soup.select('article.opportunity') or soup.select('div.opportunity') or soup.select('[class*="opportunity"]') or soup.select('li[class*="job"]') or soup.select('div[class*="job-card"]') or soup.select('div[class*="job-item"]') or soup.select('article[class*="job"]') or soup.select('div[class*="oferta"]') or soup.select('div[class*="aviso"]') or [] ) if not cards: log.debug("No se encontraron cards con selectores conocidos; usando fallback de links") seen_ids = set() for a_tag in soup.select('a[href*="/trabajo/"]'): href = a_tag.get('href', '') m = re.search(r'/trabajo/(\d+)', href) if not m: continue job_id = m.group(1) if job_id in seen_ids: continue seen_ids.add(job_id) full_url = href if href.startswith('http') else BASE_URL + href titulo = a_tag.get_text(strip=True) or None results.append({ 'job_id': job_id, 'titulo': titulo, 'empresa': None, 'ubicacion': None, 'url': full_url, 'fecha_pub': None, }) return results seen_ids = set() for card in cards: try: a_tag = ( card.select_one('a[href*="/trabajo/"]') or card.select_one('a[href]') ) if not a_tag: continue href = a_tag.get('href', '') m = re.search(r'/trabajo/(\d+)', href) if not m: continue job_id = m.group(1) if job_id in seen_ids: continue seen_ids.add(job_id) full_url = href if href.startswith('http') else BASE_URL + href titulo_tag = ( card.select_one('h1') or card.select_one('h2') or card.select_one('h3') or card.select_one('[class*="title"]') or card.select_one('[class*="titulo"]') or card.select_one('[class*="cargo"]') or card.select_one('[class*="puesto"]') ) titulo = titulo_tag.get_text(strip=True) if titulo_tag else ( a_tag.get_text(strip=True) or None ) empresa_tag = ( card.select_one('[class*="company"]') or card.select_one('[class*="empresa"]') or card.select_one('[class*="employer"]') or card.select_one('[class*="razon"]') or card.select_one('span.company') or card.select_one('div.company') ) empresa = empresa_tag.get_text(strip=True) if empresa_tag else None ubicacion_tag = ( card.select_one('[class*="location"]') or card.select_one('[class*="ubicacion"]') or card.select_one('[class*="ciudad"]') or card.select_one('[class*="region"]') or card.select_one('[class*="lugar"]') or card.select_one('span.location') or card.select_one('div.location') ) ubicacion = ubicacion_tag.get_text(strip=True) if ubicacion_tag else None fecha_pub = None fecha_tag = ( card.select_one('time') or card.select_one('[class*="date"]') or card.select_one('[class*="fecha"]') or card.select_one('[class*="publicad"]') or card.select_one('[datetime]') ) if fecha_tag: fecha_pub = ( fecha_tag.get('datetime') or fecha_tag.get_text(strip=True) or None ) results.append({ 'job_id': job_id, 'titulo': titulo, 'empresa': empresa, 'ubicacion': ubicacion, 'url': full_url, 'fecha_pub': fecha_pub, }) except Exception as exc: log.debug("Error parseando card individual: %s", exc) continue return results def main(): log.info("=== Iniciando scraper Trabajando.cl ===") sess = get_session() try: conn = psycopg2.connect( host=DB_HOST, port=DB_PORT, dbname=DB_NAME, user=DB_USER, password=DB_PASS, ) except Exception as exc: raise RuntimeError( f"No se pudo conectar a la base de datos '{DB_NAME}': {exc}" ) from exc log.info("Conexion a DB '%s' establecida.", DB_NAME) create_table(conn) total_new = 0 total_upd = 0 no_new_streak = 0 for page in range(1, MAX_PAGES + 1): url = LIST_URL.format(page=page) log.info("Procesando pagina %s/%s: %s", page, MAX_PAGES, url) try: rows = scrape_page(sess, url) except Exception as exc: log.error("Error inesperado en scrape_page pagina %s: %s", page, exc) rows = [] if not rows: log.info("Pagina %s: sin resultados - racha sin nuevos: %s/%s", page, no_new_streak + 1, NEW_STREAK) no_new_streak += 1 else: try: new_p, upd_p = upsert(conn, rows) except Exception as exc: log.error("Error en upsert pagina %s: %s", page, exc) raise RuntimeError( f"Error critico al insertar datos en DB (pagina {page}): {exc}" ) from exc log.info( "Pagina %s: %s encontrados -> %s nuevos, %s actualizados", page, len(rows), new_p, upd_p ) total_new += new_p total_upd += upd_p if new_p == 0: no_new_streak += 1 else: no_new_streak = 0 if no_new_streak >= NEW_STREAK: log.info( "Racha de %s paginas consecutivas sin nuevos registros - deteniendo.", NEW_STREAK ) break if page < MAX_PAGES: time.sleep(DELAY) try: conn.close() except Exception: pass print(f"Finalizado: {total_new} nuevos, {total_upd} actualizados") if __name__ == '__main__': main()