diff --git a/empleos/trabajando_cl/scraper.py b/empleos/trabajando_cl/scraper.py new file mode 100644 index 0000000..5c66010 --- /dev/null +++ b/empleos/trabajando_cl/scraper.py @@ -0,0 +1,381 @@ +import os +import re +import time +import logging +import requests +from bs4 import BeautifulSoup +import psycopg2 +from psycopg2.extras import execute_values +from datetime import datetime + +# -- CONFIG ------------------------------------------------------------------- +PROXY = 'socks5h://127.0.0.1:1090' +PROXIES = {'http': PROXY, 'https': PROXY} + +DB_HOST = '100.75.240.87' +DB_PORT = 5432 +DB_NAME = 'trabajando_cl' +DB_USER = 'pgadmin' +DB_PASS = 'J5BVlq65JvedWxZVrcY96OQX' + +BASE_URL = 'https://www.trabajando.cl' +LIST_URL = 'https://www.trabajando.cl/trabajo?page={page}' +UA = ( + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/124.0.0.0 Safari/537.36' +) +DELAY = 0.8 +MAX_PAGES = 20 +NEW_STREAK = 3 + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s %(levelname)s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', +) +log = logging.getLogger(__name__) + + +def get_session(): + sess = requests.Session() + sess.proxies.update(PROXIES) + sess.headers.update({ + 'User-Agent': UA, + 'Accept-Language': 'es-CL,es;q=0.9', + 'Accept': ( + 'text/html,application/xhtml+xml,' + 'application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' + ), + 'Accept-Encoding': 'gzip, deflate, br', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1', + 'Cache-Control': 'max-age=0', + }) + return sess + + +def create_table(conn): + ddl_create = """ + CREATE TABLE IF NOT EXISTS jobs ( + job_id TEXT PRIMARY KEY, + titulo TEXT, + empresa TEXT, + ubicacion TEXT, + url TEXT UNIQUE, + fecha_pub TEXT, + scraped_at TIMESTAMP DEFAULT NOW() + ); + """ + columns = [ + ('titulo', 'TEXT'), + ('empresa', 'TEXT'), + ('ubicacion', 'TEXT'), + ('url', 'TEXT'), + ('fecha_pub', 'TEXT'), + ('scraped_at', 'TIMESTAMP DEFAULT NOW()'), + ] + with conn.cursor() as cur: + cur.execute(ddl_create) + for col, typedef in columns: + try: + cur.execute( + f"ALTER TABLE jobs ADD COLUMN IF NOT EXISTS {col} {typedef};" + ) + except Exception as exc: + log.debug("ALTER TABLE ignorado para columna %s: %s", col, exc) + conn.commit() + log.info("Tabla jobs verificada/creada correctamente.") + + +def upsert(conn, rows): + if not rows: + return 0, 0 + + job_ids = [r['job_id'] for r in rows] + + with conn.cursor() as cur: + cur.execute( + "SELECT job_id FROM jobs WHERE job_id = ANY(%s);", + (job_ids,) + ) + existing = {row[0] for row in cur.fetchall()} + + tuples = [ + ( + r['job_id'], + r.get('titulo'), + r.get('empresa'), + r.get('ubicacion'), + r.get('url'), + r.get('fecha_pub'), + ) + for r in rows + ] + + sql = """ + INSERT INTO jobs (job_id, titulo, empresa, ubicacion, url, fecha_pub) + VALUES %s + ON CONFLICT (job_id) DO UPDATE SET + titulo = EXCLUDED.titulo, + empresa = EXCLUDED.empresa, + ubicacion = EXCLUDED.ubicacion, + url = EXCLUDED.url, + fecha_pub = EXCLUDED.fecha_pub, + scraped_at = NOW(); + """ + execute_values(cur, sql, tuples) + conn.commit() + + inserted = sum(1 for r in rows if r['job_id'] not in existing) + updated = len(rows) - inserted + return inserted, updated + + +def scrape_page(sess, url): + backoffs = [2, 5, 15] + resp = None + + for attempt, wait in enumerate(backoffs, 1): + try: + resp = sess.get(url, timeout=30) + + if resp.status_code in (403, 429): + log.warning( + "HTTP %s en %s (intento %s) - sleep 30s y reintento extra", + resp.status_code, url, attempt + ) + time.sleep(30) + resp = sess.get(url, timeout=30) + if resp.status_code in (403, 429): + log.warning( + "HTTP %s persiste tras reintento extra - devolviendo []", + resp.status_code + ) + return [] + + resp.raise_for_status() + break + + except requests.RequestException as exc: + log.warning( + "Error intento %s/%s en %s: %s", + attempt, len(backoffs), url, exc + ) + if attempt < len(backoffs): + time.sleep(wait) + else: + log.error("Todos los intentos fallaron para %s", url) + return [] + + if resp is None or not resp.text.strip(): + log.warning("Respuesta vacia para %s", url) + return [] + + soup = BeautifulSoup(resp.text, 'lxml') + results = [] + + cards = ( + soup.select('article.opportunity') + or soup.select('div.opportunity') + or soup.select('[class*="opportunity"]') + or soup.select('li[class*="job"]') + or soup.select('div[class*="job-card"]') + or soup.select('div[class*="job-item"]') + or soup.select('article[class*="job"]') + or soup.select('div[class*="oferta"]') + or soup.select('div[class*="aviso"]') + or [] + ) + + if not cards: + log.debug("No se encontraron cards con selectores conocidos; usando fallback de links") + seen_ids = set() + for a_tag in soup.select('a[href*="/trabajo/"]'): + href = a_tag.get('href', '') + m = re.search(r'/trabajo/(\d+)', href) + if not m: + continue + job_id = m.group(1) + if job_id in seen_ids: + continue + seen_ids.add(job_id) + full_url = href if href.startswith('http') else BASE_URL + href + titulo = a_tag.get_text(strip=True) or None + results.append({ + 'job_id': job_id, + 'titulo': titulo, + 'empresa': None, + 'ubicacion': None, + 'url': full_url, + 'fecha_pub': None, + }) + return results + + seen_ids = set() + for card in cards: + try: + a_tag = ( + card.select_one('a[href*="/trabajo/"]') + or card.select_one('a[href]') + ) + if not a_tag: + continue + + href = a_tag.get('href', '') + m = re.search(r'/trabajo/(\d+)', href) + if not m: + continue + + job_id = m.group(1) + if job_id in seen_ids: + continue + seen_ids.add(job_id) + + full_url = href if href.startswith('http') else BASE_URL + href + + titulo_tag = ( + card.select_one('h1') + or card.select_one('h2') + or card.select_one('h3') + or card.select_one('[class*="title"]') + or card.select_one('[class*="titulo"]') + or card.select_one('[class*="cargo"]') + or card.select_one('[class*="puesto"]') + ) + titulo = titulo_tag.get_text(strip=True) if titulo_tag else ( + a_tag.get_text(strip=True) or None + ) + + empresa_tag = ( + card.select_one('[class*="company"]') + or card.select_one('[class*="empresa"]') + or card.select_one('[class*="employer"]') + or card.select_one('[class*="razon"]') + or card.select_one('span.company') + or card.select_one('div.company') + ) + empresa = empresa_tag.get_text(strip=True) if empresa_tag else None + + ubicacion_tag = ( + card.select_one('[class*="location"]') + or card.select_one('[class*="ubicacion"]') + or card.select_one('[class*="ciudad"]') + or card.select_one('[class*="region"]') + or card.select_one('[class*="lugar"]') + or card.select_one('span.location') + or card.select_one('div.location') + ) + ubicacion = ubicacion_tag.get_text(strip=True) if ubicacion_tag else None + + fecha_pub = None + fecha_tag = ( + card.select_one('time') + or card.select_one('[class*="date"]') + or card.select_one('[class*="fecha"]') + or card.select_one('[class*="publicad"]') + or card.select_one('[datetime]') + ) + if fecha_tag: + fecha_pub = ( + fecha_tag.get('datetime') + or fecha_tag.get_text(strip=True) + or None + ) + + results.append({ + 'job_id': job_id, + 'titulo': titulo, + 'empresa': empresa, + 'ubicacion': ubicacion, + 'url': full_url, + 'fecha_pub': fecha_pub, + }) + + except Exception as exc: + log.debug("Error parseando card individual: %s", exc) + continue + + return results + + +def main(): + log.info("=== Iniciando scraper Trabajando.cl ===") + + sess = get_session() + + try: + conn = psycopg2.connect( + host=DB_HOST, + port=DB_PORT, + dbname=DB_NAME, + user=DB_USER, + password=DB_PASS, + ) + except Exception as exc: + raise RuntimeError( + f"No se pudo conectar a la base de datos '{DB_NAME}': {exc}" + ) from exc + + log.info("Conexion a DB '%s' establecida.", DB_NAME) + create_table(conn) + + total_new = 0 + total_upd = 0 + no_new_streak = 0 + + for page in range(1, MAX_PAGES + 1): + url = LIST_URL.format(page=page) + log.info("Procesando pagina %s/%s: %s", page, MAX_PAGES, url) + + try: + rows = scrape_page(sess, url) + except Exception as exc: + log.error("Error inesperado en scrape_page pagina %s: %s", page, exc) + rows = [] + + if not rows: + log.info("Pagina %s: sin resultados - racha sin nuevos: %s/%s", + page, no_new_streak + 1, NEW_STREAK) + no_new_streak += 1 + else: + try: + new_p, upd_p = upsert(conn, rows) + except Exception as exc: + log.error("Error en upsert pagina %s: %s", page, exc) + raise RuntimeError( + f"Error critico al insertar datos en DB (pagina {page}): {exc}" + ) from exc + + log.info( + "Pagina %s: %s encontrados -> %s nuevos, %s actualizados", + page, len(rows), new_p, upd_p + ) + total_new += new_p + total_upd += upd_p + + if new_p == 0: + no_new_streak += 1 + else: + no_new_streak = 0 + + if no_new_streak >= NEW_STREAK: + log.info( + "Racha de %s paginas consecutivas sin nuevos registros - deteniendo.", + NEW_STREAK + ) + break + + if page < MAX_PAGES: + time.sleep(DELAY) + + try: + conn.close() + except Exception: + pass + + print(f"Finalizado: {total_new} nuevos, {total_upd} actualizados") + + +if __name__ == '__main__': + main() \ No newline at end of file