portales-hispanohablantes/empleos/elempleo_co/scraper.py

#!/usr/bin/env python3
"""
Scraper ElEmpleo.com Colombia -> postgres17-central / elempleo_co
URL: /co/ofertas-empleo/?pagina=N (SSR, 20 links de trabajo por pagina)
ID: numero al final de /co/ofertas-trabajo/{slug}-{id}
"""
import os, re, time, logging
import requests
import psycopg2
from psycopg2.extras import execute_values
from datetime import datetime

logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
log = logging.getLogger('elempleo')

PROXY    = 'socks5h://127.0.0.1:1090'
PROXIES  = {'http': PROXY, 'https': PROXY}
DB_HOST  = '100.75.240.87'
DB_NAME  = 'elempleo_co'
DB_USER  = 'pgadmin'
DB_PASS  = 'J5BVlq65JvedWxZVrcY96OQX'
UA       = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36'
BASE     = 'https://www.elempleo.com'
LIST_URL = BASE + '/co/ofertas-empleo/?pagina={}'
DELAY    = 0.8
MAX_PAGES = int(os.environ.get('EL_MAX_PAGES', '30'))
NEW_STREAK = int(os.environ.get('EL_STREAK', '3'))


def get_session():
    s = requests.Session()
    s.proxies.update(PROXIES)
    s.headers.update({
        'User-Agent': UA,
        'Accept': 'text/html,application/xhtml+xml,*/*;q=0.8',
        'Accept-Language': 'es-CO,es;q=0.9',
    })
    return s


def get_conn():
    return psycopg2.connect(host=DB_HOST, port=5432, dbname=DB_NAME,
                            user=DB_USER, password=DB_PASS, connect_timeout=10)


def create_table(conn):
    cur = conn.cursor()
    cur.execute("""
        CREATE TABLE IF NOT EXISTS jobs (
            job_id     TEXT PRIMARY KEY,
            titulo     TEXT,
            empresa    TEXT,
            ubicacion  TEXT,
            url        TEXT UNIQUE,
            fecha_pub  TEXT,
            categoria  TEXT,
            scraped_at TIMESTAMP DEFAULT NOW()
        )
    """)
    conn.commit()
    cur.close()


def upsert(conn, rows):
    if not rows:
        return 0
    cur = conn.cursor()
    execute_values(cur,
        """INSERT INTO jobs (job_id, titulo, empresa, ubicacion, url, scraped_at)
           VALUES %s
           ON CONFLICT (job_id) DO UPDATE SET
             titulo=EXCLUDED.titulo, scraped_at=EXCLUDED.scraped_at""",
        [(r['job_id'], r['titulo'], r['empresa'], r['ubicacion'],
          r['url'], datetime.utcnow()) for r in rows]
    )
    n = cur.rowcount
    conn.commit()
    cur.close()
    return n


def fetch_page(sess, page):
    """Devuelve lista de dicts con {job_id, titulo, empresa, ubicacion, url}."""
    url = LIST_URL.format(page)
    for attempt in range(3):
        try:
            resp = sess.get(url, timeout=30)
            if resp.status_code == 404:
                return None  # fin
            if resp.status_code in (403, 429):
                log.warning('HTTP %d en %s (intento %d/3)', resp.status_code, url, attempt+1)
                time.sleep(30)
                continue
            resp.raise_for_status()
            break
        except Exception as e:
            if attempt == 2:
                log.error('Error tras 3 intentos en %s: %s', url, e)
                return []
            time.sleep([2, 5, 15][attempt])

    html = resp.text
    # Extraer todos los links de ofertas de trabajo
    links = list(dict.fromkeys(re.findall(r'href="(/co/ofertas-trabajo/[^"]{10,150})"', html)))
    if not links:
        return []

    jobs = []
    for href in links:
        # ID: numero al final del slug
        m = re.search(r'-(\d{7,})$', href)
        if not m:
            continue
        job_id = m.group(1)
        # Titulo: del slug (antes del ID)
        slug = href.split('/')[-1]
        titulo_raw = slug[:-(len(job_id)+1)].replace('-', ' ').strip()
        titulo = titulo_raw.title() if titulo_raw else 'Sin titulo'
        jobs.append({
            'job_id':   job_id,
            'titulo':   titulo[:300],
            'empresa':  '',
            'ubicacion': 'Colombia',
            'url':       BASE + href,
        })

    log.info('Pagina %d: %d ofertas extraidas de %s', page, len(jobs), url)
    return jobs


def main():
    log.info('=== Iniciando scraper ElEmpleo Colombia ===')
    sess = get_session()
    conn = get_conn()
    create_table(conn)

    total_new = 0
    total_upd = 0
    streak    = 0

    for page in range(1, MAX_PAGES + 1):
        jobs = fetch_page(sess, page)
        if jobs is None:
            log.info('Pagina %d: HTTP 404, fin de paginacion', page)
            break
        if not jobs:
            streak += 1
            log.info('Pagina %d: sin resultados (racha %d/%d)', page, streak, NEW_STREAK)
            if streak >= NEW_STREAK:
                break
            time.sleep(DELAY)
            continue

        n = upsert(conn, jobs)
        nuevos = n if n > 0 else 0
        streak = 0 if nuevos > 0 else streak + 1
        total_new += nuevos
        log.info('Pagina %d: %d procesados, %d nuevos/actualizados', page, len(jobs), nuevos)

        if streak >= NEW_STREAK:
            log.info('Deteniendo: %d paginas sin nuevos', streak)
            break
        time.sleep(DELAY)

    conn.close()
    print(f'Finalizado: {total_new} nuevos, {total_upd} actualizados')
    log.info('Finalizado: %d nuevos, %d actualizados', total_new, total_upd)


if __name__ == '__main__':
    main()