diff --git a/empleos/elempleo_co/scraper.py b/empleos/elempleo_co/scraper.py new file mode 100644 index 0000000..bab5572 --- /dev/null +++ b/empleos/elempleo_co/scraper.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +""" +Scraper ElEmpleo.com Colombia -> postgres17-central / elempleo_co +URL: /co/ofertas-empleo/?pagina=N (SSR, 20 links de trabajo por pagina) +ID: numero al final de /co/ofertas-trabajo/{slug}-{id} +""" +import os, re, time, logging +import requests +import psycopg2 +from psycopg2.extras import execute_values +from datetime import datetime + +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') +log = logging.getLogger('elempleo') + +PROXY = 'socks5h://127.0.0.1:1090' +PROXIES = {'http': PROXY, 'https': PROXY} +DB_HOST = '100.75.240.87' +DB_NAME = 'elempleo_co' +DB_USER = 'pgadmin' +DB_PASS = 'J5BVlq65JvedWxZVrcY96OQX' +UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36' +BASE = 'https://www.elempleo.com' +LIST_URL = BASE + '/co/ofertas-empleo/?pagina={}' +DELAY = 0.8 +MAX_PAGES = int(os.environ.get('EL_MAX_PAGES', '30')) +NEW_STREAK = int(os.environ.get('EL_STREAK', '3')) + + +def get_session(): + s = requests.Session() + s.proxies.update(PROXIES) + s.headers.update({ + 'User-Agent': UA, + 'Accept': 'text/html,application/xhtml+xml,*/*;q=0.8', + 'Accept-Language': 'es-CO,es;q=0.9', + }) + return s + + +def get_conn(): + return psycopg2.connect(host=DB_HOST, port=5432, dbname=DB_NAME, + user=DB_USER, password=DB_PASS, connect_timeout=10) + + +def create_table(conn): + cur = conn.cursor() + cur.execute(""" + CREATE TABLE IF NOT EXISTS jobs ( + job_id TEXT PRIMARY KEY, + titulo TEXT, + empresa TEXT, + ubicacion TEXT, + url TEXT UNIQUE, + fecha_pub TEXT, + categoria TEXT, + scraped_at TIMESTAMP DEFAULT NOW() + ) + """) + conn.commit() + cur.close() + + +def upsert(conn, rows): + if not rows: + return 0 + cur = conn.cursor() + execute_values(cur, + """INSERT INTO jobs (job_id, titulo, empresa, ubicacion, url, scraped_at) + VALUES %s + ON CONFLICT (job_id) DO UPDATE SET + titulo=EXCLUDED.titulo, scraped_at=EXCLUDED.scraped_at""", + [(r['job_id'], r['titulo'], r['empresa'], r['ubicacion'], + r['url'], datetime.utcnow()) for r in rows] + ) + n = cur.rowcount + conn.commit() + cur.close() + return n + + +def fetch_page(sess, page): + """Devuelve lista de dicts con {job_id, titulo, empresa, ubicacion, url}.""" + url = LIST_URL.format(page) + for attempt in range(3): + try: + resp = sess.get(url, timeout=30) + if resp.status_code == 404: + return None # fin + if resp.status_code in (403, 429): + log.warning('HTTP %d en %s (intento %d/3)', resp.status_code, url, attempt+1) + time.sleep(30) + continue + resp.raise_for_status() + break + except Exception as e: + if attempt == 2: + log.error('Error tras 3 intentos en %s: %s', url, e) + return [] + time.sleep([2, 5, 15][attempt]) + + html = resp.text + # Extraer todos los links de ofertas de trabajo + links = list(dict.fromkeys(re.findall(r'href="(/co/ofertas-trabajo/[^"]{10,150})"', html))) + if not links: + return [] + + jobs = [] + for href in links: + # ID: numero al final del slug + m = re.search(r'-(\d{7,})$', href) + if not m: + continue + job_id = m.group(1) + # Titulo: del slug (antes del ID) + slug = href.split('/')[-1] + titulo_raw = slug[:-(len(job_id)+1)].replace('-', ' ').strip() + titulo = titulo_raw.title() if titulo_raw else 'Sin titulo' + jobs.append({ + 'job_id': job_id, + 'titulo': titulo[:300], + 'empresa': '', + 'ubicacion': 'Colombia', + 'url': BASE + href, + }) + + log.info('Pagina %d: %d ofertas extraidas de %s', page, len(jobs), url) + return jobs + + +def main(): + log.info('=== Iniciando scraper ElEmpleo Colombia ===') + sess = get_session() + conn = get_conn() + create_table(conn) + + total_new = 0 + total_upd = 0 + streak = 0 + + for page in range(1, MAX_PAGES + 1): + jobs = fetch_page(sess, page) + if jobs is None: + log.info('Pagina %d: HTTP 404, fin de paginacion', page) + break + if not jobs: + streak += 1 + log.info('Pagina %d: sin resultados (racha %d/%d)', page, streak, NEW_STREAK) + if streak >= NEW_STREAK: + break + time.sleep(DELAY) + continue + + n = upsert(conn, jobs) + nuevos = n if n > 0 else 0 + streak = 0 if nuevos > 0 else streak + 1 + total_new += nuevos + log.info('Pagina %d: %d procesados, %d nuevos/actualizados', page, len(jobs), nuevos) + + if streak >= NEW_STREAK: + log.info('Deteniendo: %d paginas sin nuevos', streak) + break + time.sleep(DELAY) + + conn.close() + print(f'Finalizado: {total_new} nuevos, {total_upd} actualizados') + log.info('Finalizado: %d nuevos, %d actualizados', total_new, total_upd) + + +if __name__ == '__main__': + main()