381 lines
11 KiB
Python
381 lines
11 KiB
Python
import os
|
|
import re
|
|
import time
|
|
import logging
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
import psycopg2
|
|
from psycopg2.extras import execute_values
|
|
from datetime import datetime
|
|
|
|
# -- CONFIG -------------------------------------------------------------------
|
|
PROXY = 'socks5h://127.0.0.1:1090'
|
|
PROXIES = {'http': PROXY, 'https': PROXY}
|
|
|
|
DB_HOST = '100.75.240.87'
|
|
DB_PORT = 5432
|
|
DB_NAME = 'trabajando_cl'
|
|
DB_USER = 'pgadmin'
|
|
DB_PASS = 'J5BVlq65JvedWxZVrcY96OQX'
|
|
|
|
BASE_URL = 'https://www.trabajando.cl'
|
|
LIST_URL = 'https://www.trabajando.cl/trabajo?page={page}'
|
|
UA = (
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
|
|
'AppleWebKit/537.36 (KHTML, like Gecko) '
|
|
'Chrome/124.0.0.0 Safari/537.36'
|
|
)
|
|
DELAY = 0.8
|
|
MAX_PAGES = 20
|
|
NEW_STREAK = 3
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s %(levelname)s %(message)s',
|
|
datefmt='%Y-%m-%d %H:%M:%S',
|
|
)
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def get_session():
|
|
sess = requests.Session()
|
|
sess.proxies.update(PROXIES)
|
|
sess.headers.update({
|
|
'User-Agent': UA,
|
|
'Accept-Language': 'es-CL,es;q=0.9',
|
|
'Accept': (
|
|
'text/html,application/xhtml+xml,'
|
|
'application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'
|
|
),
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
'Connection': 'keep-alive',
|
|
'Upgrade-Insecure-Requests': '1',
|
|
'Cache-Control': 'max-age=0',
|
|
})
|
|
return sess
|
|
|
|
|
|
def create_table(conn):
|
|
ddl_create = """
|
|
CREATE TABLE IF NOT EXISTS jobs (
|
|
job_id TEXT PRIMARY KEY,
|
|
titulo TEXT,
|
|
empresa TEXT,
|
|
ubicacion TEXT,
|
|
url TEXT UNIQUE,
|
|
fecha_pub TEXT,
|
|
scraped_at TIMESTAMP DEFAULT NOW()
|
|
);
|
|
"""
|
|
columns = [
|
|
('titulo', 'TEXT'),
|
|
('empresa', 'TEXT'),
|
|
('ubicacion', 'TEXT'),
|
|
('url', 'TEXT'),
|
|
('fecha_pub', 'TEXT'),
|
|
('scraped_at', 'TIMESTAMP DEFAULT NOW()'),
|
|
]
|
|
with conn.cursor() as cur:
|
|
cur.execute(ddl_create)
|
|
for col, typedef in columns:
|
|
try:
|
|
cur.execute(
|
|
f"ALTER TABLE jobs ADD COLUMN IF NOT EXISTS {col} {typedef};"
|
|
)
|
|
except Exception as exc:
|
|
log.debug("ALTER TABLE ignorado para columna %s: %s", col, exc)
|
|
conn.commit()
|
|
log.info("Tabla jobs verificada/creada correctamente.")
|
|
|
|
|
|
def upsert(conn, rows):
|
|
if not rows:
|
|
return 0, 0
|
|
|
|
job_ids = [r['job_id'] for r in rows]
|
|
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"SELECT job_id FROM jobs WHERE job_id = ANY(%s);",
|
|
(job_ids,)
|
|
)
|
|
existing = {row[0] for row in cur.fetchall()}
|
|
|
|
tuples = [
|
|
(
|
|
r['job_id'],
|
|
r.get('titulo'),
|
|
r.get('empresa'),
|
|
r.get('ubicacion'),
|
|
r.get('url'),
|
|
r.get('fecha_pub'),
|
|
)
|
|
for r in rows
|
|
]
|
|
|
|
sql = """
|
|
INSERT INTO jobs (job_id, titulo, empresa, ubicacion, url, fecha_pub)
|
|
VALUES %s
|
|
ON CONFLICT (job_id) DO UPDATE SET
|
|
titulo = EXCLUDED.titulo,
|
|
empresa = EXCLUDED.empresa,
|
|
ubicacion = EXCLUDED.ubicacion,
|
|
url = EXCLUDED.url,
|
|
fecha_pub = EXCLUDED.fecha_pub,
|
|
scraped_at = NOW();
|
|
"""
|
|
execute_values(cur, sql, tuples)
|
|
conn.commit()
|
|
|
|
inserted = sum(1 for r in rows if r['job_id'] not in existing)
|
|
updated = len(rows) - inserted
|
|
return inserted, updated
|
|
|
|
|
|
def scrape_page(sess, url):
|
|
backoffs = [2, 5, 15]
|
|
resp = None
|
|
|
|
for attempt, wait in enumerate(backoffs, 1):
|
|
try:
|
|
resp = sess.get(url, timeout=30)
|
|
|
|
if resp.status_code in (403, 429):
|
|
log.warning(
|
|
"HTTP %s en %s (intento %s) - sleep 30s y reintento extra",
|
|
resp.status_code, url, attempt
|
|
)
|
|
time.sleep(30)
|
|
resp = sess.get(url, timeout=30)
|
|
if resp.status_code in (403, 429):
|
|
log.warning(
|
|
"HTTP %s persiste tras reintento extra - devolviendo []",
|
|
resp.status_code
|
|
)
|
|
return []
|
|
|
|
resp.raise_for_status()
|
|
break
|
|
|
|
except requests.RequestException as exc:
|
|
log.warning(
|
|
"Error intento %s/%s en %s: %s",
|
|
attempt, len(backoffs), url, exc
|
|
)
|
|
if attempt < len(backoffs):
|
|
time.sleep(wait)
|
|
else:
|
|
log.error("Todos los intentos fallaron para %s", url)
|
|
return []
|
|
|
|
if resp is None or not resp.text.strip():
|
|
log.warning("Respuesta vacia para %s", url)
|
|
return []
|
|
|
|
soup = BeautifulSoup(resp.text, 'lxml')
|
|
results = []
|
|
|
|
cards = (
|
|
soup.select('article.opportunity')
|
|
or soup.select('div.opportunity')
|
|
or soup.select('[class*="opportunity"]')
|
|
or soup.select('li[class*="job"]')
|
|
or soup.select('div[class*="job-card"]')
|
|
or soup.select('div[class*="job-item"]')
|
|
or soup.select('article[class*="job"]')
|
|
or soup.select('div[class*="oferta"]')
|
|
or soup.select('div[class*="aviso"]')
|
|
or []
|
|
)
|
|
|
|
if not cards:
|
|
log.debug("No se encontraron cards con selectores conocidos; usando fallback de links")
|
|
seen_ids = set()
|
|
for a_tag in soup.select('a[href*="/trabajo/"]'):
|
|
href = a_tag.get('href', '')
|
|
m = re.search(r'/trabajo/(\d+)', href)
|
|
if not m:
|
|
continue
|
|
job_id = m.group(1)
|
|
if job_id in seen_ids:
|
|
continue
|
|
seen_ids.add(job_id)
|
|
full_url = href if href.startswith('http') else BASE_URL + href
|
|
titulo = a_tag.get_text(strip=True) or None
|
|
results.append({
|
|
'job_id': job_id,
|
|
'titulo': titulo,
|
|
'empresa': None,
|
|
'ubicacion': None,
|
|
'url': full_url,
|
|
'fecha_pub': None,
|
|
})
|
|
return results
|
|
|
|
seen_ids = set()
|
|
for card in cards:
|
|
try:
|
|
a_tag = (
|
|
card.select_one('a[href*="/trabajo/"]')
|
|
or card.select_one('a[href]')
|
|
)
|
|
if not a_tag:
|
|
continue
|
|
|
|
href = a_tag.get('href', '')
|
|
m = re.search(r'/trabajo/(\d+)', href)
|
|
if not m:
|
|
continue
|
|
|
|
job_id = m.group(1)
|
|
if job_id in seen_ids:
|
|
continue
|
|
seen_ids.add(job_id)
|
|
|
|
full_url = href if href.startswith('http') else BASE_URL + href
|
|
|
|
titulo_tag = (
|
|
card.select_one('h1')
|
|
or card.select_one('h2')
|
|
or card.select_one('h3')
|
|
or card.select_one('[class*="title"]')
|
|
or card.select_one('[class*="titulo"]')
|
|
or card.select_one('[class*="cargo"]')
|
|
or card.select_one('[class*="puesto"]')
|
|
)
|
|
titulo = titulo_tag.get_text(strip=True) if titulo_tag else (
|
|
a_tag.get_text(strip=True) or None
|
|
)
|
|
|
|
empresa_tag = (
|
|
card.select_one('[class*="company"]')
|
|
or card.select_one('[class*="empresa"]')
|
|
or card.select_one('[class*="employer"]')
|
|
or card.select_one('[class*="razon"]')
|
|
or card.select_one('span.company')
|
|
or card.select_one('div.company')
|
|
)
|
|
empresa = empresa_tag.get_text(strip=True) if empresa_tag else None
|
|
|
|
ubicacion_tag = (
|
|
card.select_one('[class*="location"]')
|
|
or card.select_one('[class*="ubicacion"]')
|
|
or card.select_one('[class*="ciudad"]')
|
|
or card.select_one('[class*="region"]')
|
|
or card.select_one('[class*="lugar"]')
|
|
or card.select_one('span.location')
|
|
or card.select_one('div.location')
|
|
)
|
|
ubicacion = ubicacion_tag.get_text(strip=True) if ubicacion_tag else None
|
|
|
|
fecha_pub = None
|
|
fecha_tag = (
|
|
card.select_one('time')
|
|
or card.select_one('[class*="date"]')
|
|
or card.select_one('[class*="fecha"]')
|
|
or card.select_one('[class*="publicad"]')
|
|
or card.select_one('[datetime]')
|
|
)
|
|
if fecha_tag:
|
|
fecha_pub = (
|
|
fecha_tag.get('datetime')
|
|
or fecha_tag.get_text(strip=True)
|
|
or None
|
|
)
|
|
|
|
results.append({
|
|
'job_id': job_id,
|
|
'titulo': titulo,
|
|
'empresa': empresa,
|
|
'ubicacion': ubicacion,
|
|
'url': full_url,
|
|
'fecha_pub': fecha_pub,
|
|
})
|
|
|
|
except Exception as exc:
|
|
log.debug("Error parseando card individual: %s", exc)
|
|
continue
|
|
|
|
return results
|
|
|
|
|
|
def main():
|
|
log.info("=== Iniciando scraper Trabajando.cl ===")
|
|
|
|
sess = get_session()
|
|
|
|
try:
|
|
conn = psycopg2.connect(
|
|
host=DB_HOST,
|
|
port=DB_PORT,
|
|
dbname=DB_NAME,
|
|
user=DB_USER,
|
|
password=DB_PASS,
|
|
)
|
|
except Exception as exc:
|
|
raise RuntimeError(
|
|
f"No se pudo conectar a la base de datos '{DB_NAME}': {exc}"
|
|
) from exc
|
|
|
|
log.info("Conexion a DB '%s' establecida.", DB_NAME)
|
|
create_table(conn)
|
|
|
|
total_new = 0
|
|
total_upd = 0
|
|
no_new_streak = 0
|
|
|
|
for page in range(1, MAX_PAGES + 1):
|
|
url = LIST_URL.format(page=page)
|
|
log.info("Procesando pagina %s/%s: %s", page, MAX_PAGES, url)
|
|
|
|
try:
|
|
rows = scrape_page(sess, url)
|
|
except Exception as exc:
|
|
log.error("Error inesperado en scrape_page pagina %s: %s", page, exc)
|
|
rows = []
|
|
|
|
if not rows:
|
|
log.info("Pagina %s: sin resultados - racha sin nuevos: %s/%s",
|
|
page, no_new_streak + 1, NEW_STREAK)
|
|
no_new_streak += 1
|
|
else:
|
|
try:
|
|
new_p, upd_p = upsert(conn, rows)
|
|
except Exception as exc:
|
|
log.error("Error en upsert pagina %s: %s", page, exc)
|
|
raise RuntimeError(
|
|
f"Error critico al insertar datos en DB (pagina {page}): {exc}"
|
|
) from exc
|
|
|
|
log.info(
|
|
"Pagina %s: %s encontrados -> %s nuevos, %s actualizados",
|
|
page, len(rows), new_p, upd_p
|
|
)
|
|
total_new += new_p
|
|
total_upd += upd_p
|
|
|
|
if new_p == 0:
|
|
no_new_streak += 1
|
|
else:
|
|
no_new_streak = 0
|
|
|
|
if no_new_streak >= NEW_STREAK:
|
|
log.info(
|
|
"Racha de %s paginas consecutivas sin nuevos registros - deteniendo.",
|
|
NEW_STREAK
|
|
)
|
|
break
|
|
|
|
if page < MAX_PAGES:
|
|
time.sleep(DELAY)
|
|
|
|
try:
|
|
conn.close()
|
|
except Exception:
|
|
pass
|
|
|
|
print(f"Finalizado: {total_new} nuevos, {total_upd} actualizados")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main() |