479 lines
16 KiB
Python
479 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
Scraper: Encuentra24.com - Empleos Centroamerica (6 paises)
|
|
Paises: Panama, Costa Rica, Guatemala, El Salvador, Honduras, Nicaragua
|
|
DB: encuentra24_empleos
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import time
|
|
import logging
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
import psycopg2
|
|
from psycopg2.extras import execute_values
|
|
|
|
# -- Configuracion -------------------------------------------------------------
|
|
PROXY_URL = 'socks5h://127.0.0.1:1090'
|
|
PROXIES = {'http': PROXY_URL, 'https': PROXY_URL}
|
|
|
|
DB_HOST = '100.75.240.87'
|
|
DB_PORT = 5432
|
|
DB_NAME = 'encuentra24_empleos'
|
|
DB_USER = 'pgadmin'
|
|
DB_PASS = 'J5BVlq65JvedWxZVrcY96OQX'
|
|
|
|
UA = (
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
|
|
'AppleWebKit/537.36 (KHTML, like Gecko) '
|
|
'Chrome/124.0.0.0 Safari/537.36'
|
|
)
|
|
|
|
DELAY = 0.8 # segundos entre requests
|
|
MAX_PAGES = 15 # maximo de paginas por pais
|
|
NEW_STREAK = 2 # paginas consecutivas sin nuevos -> parar por pais
|
|
|
|
BASE_URL = 'https://www.encuentra24.com'
|
|
|
|
# Paises: (codigo_url, nombre_pais)
|
|
COUNTRIES = [
|
|
('pa', 'panama'),
|
|
('cr', 'costa-rica'),
|
|
('gt', 'guatemala'),
|
|
('sv', 'el-salvador'),
|
|
('hn', 'honduras'),
|
|
('ni', 'nicaragua'),
|
|
]
|
|
|
|
# -- Logging -------------------------------------------------------------------
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s [%(levelname)s] %(message)s',
|
|
datefmt='%Y-%m-%d %H:%M:%S',
|
|
)
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
# -- Sesion HTTP ---------------------------------------------------------------
|
|
def get_session() -> requests.Session:
|
|
"""Crea y devuelve una sesion HTTP configurada con proxy y headers."""
|
|
sess = requests.Session()
|
|
sess.proxies.update(PROXIES)
|
|
sess.headers.update({
|
|
'User-Agent': UA,
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
'Accept-Language': 'es-ES,es;q=0.9,en;q=0.8',
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
'Connection': 'keep-alive',
|
|
'Cache-Control': 'no-cache',
|
|
'Upgrade-Insecure-Requests': '1',
|
|
})
|
|
return sess
|
|
|
|
|
|
# -- Base de datos -------------------------------------------------------------
|
|
def create_table(conn) -> None:
|
|
"""Crea la tabla jobs si no existe y anade columnas faltantes."""
|
|
cur = conn.cursor()
|
|
cur.execute("""
|
|
CREATE TABLE IF NOT EXISTS jobs (
|
|
job_id TEXT PRIMARY KEY,
|
|
titulo TEXT,
|
|
empresa TEXT,
|
|
ubicacion TEXT,
|
|
url TEXT UNIQUE,
|
|
pais TEXT,
|
|
fecha_pub TEXT,
|
|
scraped_at TIMESTAMP DEFAULT NOW()
|
|
)
|
|
""")
|
|
conn.commit()
|
|
|
|
# ALTER TABLE para garantizar columnas en tablas pre-existentes
|
|
alter_stmts = [
|
|
"ALTER TABLE jobs ADD COLUMN IF NOT EXISTS titulo TEXT",
|
|
"ALTER TABLE jobs ADD COLUMN IF NOT EXISTS empresa TEXT",
|
|
"ALTER TABLE jobs ADD COLUMN IF NOT EXISTS ubicacion TEXT",
|
|
"ALTER TABLE jobs ADD COLUMN IF NOT EXISTS url TEXT",
|
|
"ALTER TABLE jobs ADD COLUMN IF NOT EXISTS pais TEXT",
|
|
"ALTER TABLE jobs ADD COLUMN IF NOT EXISTS fecha_pub TEXT",
|
|
"ALTER TABLE jobs ADD COLUMN IF NOT EXISTS scraped_at TIMESTAMP DEFAULT NOW()",
|
|
]
|
|
for stmt in alter_stmts:
|
|
try:
|
|
cur.execute(stmt)
|
|
conn.commit()
|
|
except Exception as exc:
|
|
conn.rollback()
|
|
log.debug("ALTER ignorado (columna ya existe): %s", exc)
|
|
|
|
cur.close()
|
|
log.info("Tabla 'jobs' verificada/creada. Script: %s", os.path.abspath(__file__))
|
|
|
|
|
|
def upsert(conn, rows: list) -> tuple:
|
|
"""
|
|
Inserta o actualiza filas en la tabla jobs.
|
|
Devuelve (nuevos, actualizados).
|
|
"""
|
|
if not rows:
|
|
return 0, 0
|
|
|
|
cur = conn.cursor()
|
|
|
|
# Verificar cuales job_ids ya existen para contar nuevos vs actualizados
|
|
ids = [r['job_id'] for r in rows]
|
|
cur.execute("SELECT job_id FROM jobs WHERE job_id = ANY(%s)", (ids,))
|
|
existing = {rec[0] for rec in cur.fetchall()}
|
|
|
|
nuevos = sum(1 for r in rows if r['job_id'] not in existing)
|
|
actualizados = sum(1 for r in rows if r['job_id'] in existing)
|
|
|
|
tuples = [
|
|
(r['job_id'], r['titulo'], r['empresa'],
|
|
r['ubicacion'], r['url'], r['pais'], r['fecha_pub'])
|
|
for r in rows
|
|
]
|
|
|
|
execute_values(
|
|
cur,
|
|
"""
|
|
INSERT INTO jobs (job_id, titulo, empresa, ubicacion, url, pais, fecha_pub, scraped_at)
|
|
VALUES %s
|
|
ON CONFLICT (job_id) DO UPDATE SET
|
|
titulo = EXCLUDED.titulo,
|
|
empresa = EXCLUDED.empresa,
|
|
ubicacion = EXCLUDED.ubicacion,
|
|
url = EXCLUDED.url,
|
|
pais = EXCLUDED.pais,
|
|
fecha_pub = EXCLUDED.fecha_pub,
|
|
scraped_at = NOW()
|
|
""",
|
|
tuples,
|
|
template="(%s, %s, %s, %s, %s, %s, %s, NOW())",
|
|
)
|
|
conn.commit()
|
|
cur.close()
|
|
return nuevos, actualizados
|
|
|
|
|
|
# -- Helpers de parsing --------------------------------------------------------
|
|
def _extract_text(tag, selectors: list) -> str:
|
|
"""Prueba varios selectores CSS en el tag y devuelve el primer texto hallado."""
|
|
if tag is None:
|
|
return ''
|
|
for sel in selectors:
|
|
try:
|
|
el = tag.select_one(sel)
|
|
if el:
|
|
return el.get_text(separator=' ', strip=True)
|
|
except Exception:
|
|
continue
|
|
return ''
|
|
|
|
|
|
def _extract_job_id(href: str) -> str:
|
|
"""Extrae el slug/job_id de la URL de la oferta."""
|
|
# Patron esperado: /{cc}/empleos/{slug} o /empleos/{slug}
|
|
m = re.search(r'/empleos/([^/?#]+)', href)
|
|
if m:
|
|
return m.group(1).strip()
|
|
# Fallback: ultimo segmento no vacio de la ruta
|
|
path = href.split('?')[0].split('#')[0]
|
|
segments = [s for s in path.rstrip('/').split('/') if s]
|
|
return segments[-1] if segments else ''
|
|
|
|
|
|
def _build_full_url(href: str) -> str:
|
|
"""Convierte un href relativo en URL absoluta."""
|
|
if href.startswith('http'):
|
|
return href
|
|
if href.startswith('//'):
|
|
return 'https:' + href
|
|
return BASE_URL + ('' if href.startswith('/') else '/') + href
|
|
|
|
|
|
# -- Scraping ------------------------------------------------------------------
|
|
def scrape_page(sess: requests.Session, url: str, pais: str) -> list:
|
|
"""
|
|
Descarga y parsea una pagina de listado de empleos de Encuentra24.
|
|
- 3 intentos con backoff (2s, 5s, 15s) ante errores de red.
|
|
- HTTP 403/429: espera 30s extra antes de reintentar; devuelve [] si persiste.
|
|
- Parsing robusto con multiples selectores de fallback.
|
|
Devuelve lista de dicts con los campos del job.
|
|
"""
|
|
backoffs = [2, 5, 15]
|
|
resp = None
|
|
|
|
for attempt, wait in enumerate(backoffs, start=1):
|
|
try:
|
|
resp = sess.get(url, timeout=30)
|
|
except requests.exceptions.RequestException as exc:
|
|
log.warning(
|
|
"Error de red intento %d/%d [%s]: %s",
|
|
attempt, len(backoffs), url, exc,
|
|
)
|
|
if attempt < len(backoffs):
|
|
time.sleep(wait)
|
|
continue
|
|
log.error("Agotados %d intentos de red para %s", len(backoffs), url)
|
|
return []
|
|
|
|
# Bloqueo / rate-limit
|
|
if resp.status_code in (403, 429):
|
|
log.warning(
|
|
"HTTP %d en %s (intento %d). Esperando 30s antes de reintentar.",
|
|
resp.status_code, url, attempt,
|
|
)
|
|
time.sleep(30)
|
|
if attempt >= len(backoffs):
|
|
log.warning(
|
|
"Sin mas reintentos tras HTTP %d -> devolviendo [].",
|
|
resp.status_code,
|
|
)
|
|
return []
|
|
continue
|
|
|
|
# Otros errores HTTP
|
|
if resp.status_code != 200:
|
|
log.warning("HTTP %d inesperado en %s. Devolviendo [].", resp.status_code, url)
|
|
return []
|
|
|
|
break # 200 OK
|
|
|
|
if resp is None or not resp.text.strip():
|
|
log.warning("Respuesta vacia para %s", url)
|
|
return []
|
|
|
|
soup = BeautifulSoup(resp.text, 'lxml')
|
|
jobs = []
|
|
|
|
# -- Selectores de cards (especifico -> generico) --------------------------
|
|
card_selectors = [
|
|
'div.listing-card',
|
|
'article.listing',
|
|
'div[class*="listing-card"]',
|
|
'li[class*="listing-item"]',
|
|
'div[class*="job-card"]',
|
|
'div[class*="result-item"]',
|
|
'div[class*="ad-card"]',
|
|
'div[class*="search-result"]',
|
|
'article[class*="job"]',
|
|
]
|
|
cards = []
|
|
for sel in card_selectors:
|
|
try:
|
|
found = soup.select(sel)
|
|
if found:
|
|
log.debug("Cards con selector '%s': %d", sel, len(found))
|
|
cards = found
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
# Ultimo recurso: todos los links de empleo de la pagina
|
|
if not cards:
|
|
try:
|
|
link_tags = soup.select('a[href*="/empleos/"]')
|
|
if link_tags:
|
|
log.debug("Fallback a links directos: %d encontrados", len(link_tags))
|
|
cards = link_tags
|
|
except Exception:
|
|
pass
|
|
|
|
if not cards:
|
|
log.info("Sin resultados en %s (ultima pagina?)", url)
|
|
return []
|
|
|
|
seen_ids = set() # deduplicar dentro de la misma pagina
|
|
|
|
for card in cards:
|
|
try:
|
|
# -- Link principal ------------------------------------------------
|
|
if card.name == 'a':
|
|
link_tag = card
|
|
else:
|
|
link_tag = (
|
|
card.select_one('a[href*="/empleos/"]')
|
|
or card.select_one('a')
|
|
)
|
|
|
|
href = (link_tag.get('href', '') if link_tag else '').strip()
|
|
if not href:
|
|
continue
|
|
|
|
full_url = _build_full_url(href)
|
|
|
|
# -- job_id --------------------------------------------------------
|
|
job_id = _extract_job_id(href)
|
|
if not job_id:
|
|
log.debug("job_id vacio para href=%s; ignorando.", href)
|
|
continue
|
|
if job_id in seen_ids:
|
|
continue
|
|
seen_ids.add(job_id)
|
|
|
|
# -- Titulo --------------------------------------------------------
|
|
titulo = _extract_text(card, [
|
|
'h2', 'h3', 'h1',
|
|
'[class*="title"]', '[class*="titulo"]',
|
|
'[class*="heading"]', 'p[class*="name"]',
|
|
])
|
|
# Fallback: texto del link
|
|
if not titulo and link_tag:
|
|
titulo = link_tag.get_text(separator=' ', strip=True)
|
|
titulo = titulo or 'Sin titulo'
|
|
|
|
# -- Empresa -------------------------------------------------------
|
|
empresa = _extract_text(card, [
|
|
'[class*="company"]',
|
|
'[class*="empresa"]',
|
|
'[class*="advertiser"]',
|
|
'[class*="employer"]',
|
|
'[class*="client"]',
|
|
'[class*="brand"]',
|
|
])
|
|
|
|
# -- Ubicacion -----------------------------------------------------
|
|
ubicacion = _extract_text(card, [
|
|
'[class*="location"]',
|
|
'[class*="ubicacion"]',
|
|
'[class*="ciudad"]',
|
|
'[class*="city"]',
|
|
'[class*="place"]',
|
|
'[class*="address"]',
|
|
'[class*="region"]',
|
|
])
|
|
|
|
# -- Fecha publicacion ----------------------------------------------
|
|
fecha_pub = ''
|
|
try:
|
|
f_tag = (
|
|
card.select_one('time')
|
|
or card.select_one('[datetime]')
|
|
or card.select_one('[class*="date"]')
|
|
or card.select_one('[class*="fecha"]')
|
|
or card.select_one('[class*="published"]')
|
|
or card.select_one('[class*="ago"]')
|
|
)
|
|
if f_tag:
|
|
fecha_pub = (
|
|
f_tag.get('datetime', '').strip()
|
|
or f_tag.get_text(strip=True)
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
jobs.append({
|
|
'job_id': job_id,
|
|
'titulo': titulo,
|
|
'empresa': empresa,
|
|
'ubicacion': ubicacion,
|
|
'url': full_url,
|
|
'pais': pais,
|
|
'fecha_pub': fecha_pub,
|
|
})
|
|
|
|
except Exception as exc:
|
|
log.warning("Error parseando card: %s", exc)
|
|
continue
|
|
|
|
log.info(" -> %d ofertas parseadas en %s", len(jobs), url)
|
|
return jobs
|
|
|
|
|
|
# -- Main ----------------------------------------------------------------------
|
|
def main():
|
|
total_nuevos = 0
|
|
total_actualizados = 0
|
|
|
|
log.info("=== Inicio scraper Encuentra24 Empleos ===")
|
|
|
|
# Conexion a PostgreSQL
|
|
try:
|
|
conn = psycopg2.connect(
|
|
host=DB_HOST,
|
|
port=DB_PORT,
|
|
dbname=DB_NAME,
|
|
user=DB_USER,
|
|
password=DB_PASS,
|
|
)
|
|
log.info("Conectado a PostgreSQL: %s@%s/%s", DB_USER, DB_HOST, DB_NAME)
|
|
except Exception as exc:
|
|
raise RuntimeError(f"No se pudo conectar a PostgreSQL: {exc}") from exc
|
|
|
|
try:
|
|
create_table(conn)
|
|
sess = get_session()
|
|
|
|
for cc, nombre_pais in COUNTRIES:
|
|
log.info("== Pais: %s (%s) ==", nombre_pais.upper(), cc)
|
|
paginas_sin_nuevos = 0
|
|
|
|
for page in range(1, MAX_PAGES + 1):
|
|
page_url = f"{BASE_URL}/{cc}/empleos?page={page}"
|
|
log.info("[%s] Pagina %d/%d -> %s", cc, page, MAX_PAGES, page_url)
|
|
|
|
rows = scrape_page(sess, page_url, nombre_pais)
|
|
|
|
if not rows:
|
|
log.info(
|
|
"[%s] Pagina %d sin resultados -> fin para este pais.",
|
|
cc, page,
|
|
)
|
|
break
|
|
|
|
nuevos, actualizados = upsert(conn, rows)
|
|
total_nuevos += nuevos
|
|
total_actualizados += actualizados
|
|
|
|
log.info(
|
|
"[%s] Pag %d: %d nuevos, %d actualizados (acumulado: %d/%d)",
|
|
cc, page, nuevos, actualizados,
|
|
total_nuevos, total_actualizados,
|
|
)
|
|
|
|
if nuevos == 0:
|
|
paginas_sin_nuevos += 1
|
|
log.info(
|
|
"[%s] Paginas consecutivas sin nuevos: %d/%d",
|
|
cc, paginas_sin_nuevos, NEW_STREAK,
|
|
)
|
|
if paginas_sin_nuevos >= NEW_STREAK:
|
|
log.info(
|
|
"[%s] Limite de %d paginas sin nuevos -> siguiente pais.",
|
|
cc, NEW_STREAK,
|
|
)
|
|
break
|
|
else:
|
|
paginas_sin_nuevos = 0
|
|
|
|
time.sleep(DELAY)
|
|
|
|
# Pausa entre paises para no sobrecargar el servidor
|
|
time.sleep(DELAY * 3)
|
|
|
|
except RuntimeError:
|
|
raise
|
|
except Exception as exc:
|
|
raise RuntimeError(f"Error inesperado en scraper: {exc}") from exc
|
|
finally:
|
|
try:
|
|
conn.close()
|
|
log.info("Conexion PostgreSQL cerrada.")
|
|
except Exception:
|
|
pass
|
|
|
|
# Salida requerida por el DAG para deteccion de exito
|
|
print(f"Finalizado: {total_nuevos} nuevos, {total_actualizados} actualizados")
|
|
log.info(
|
|
"=== Scraper Encuentra24 completado. Nuevos: %d | Actualizados: %d ===",
|
|
total_nuevos, total_actualizados,
|
|
)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|