From 0f63449b0a8bb98c4b045dcbb1ead80e7483be4e Mon Sep 17 00:00:00 2001 From: claude Date: Wed, 29 Jul 2026 06:49:44 +0200 Subject: [PATCH] wip: DAG encuentra24_empleos_daily --- dags/encuentra24_empleos_daily.py | 71 +++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 dags/encuentra24_empleos_daily.py diff --git a/dags/encuentra24_empleos_daily.py b/dags/encuentra24_empleos_daily.py new file mode 100644 index 0000000..4aa874d --- /dev/null +++ b/dags/encuentra24_empleos_daily.py @@ -0,0 +1,71 @@ +from airflow import DAG +from airflow.operators.python import PythonOperator +from datetime import datetime, timedelta +import paramiko + +# -- Configuracion ------------------------------------------------------------- +SSH_HOST = '152.53.242.31' +SSH_USER = 'root' +SSH_PASS = 'XpvvdHnNr0Bq9qU' +VENV = '/home/jpuma/proyectos/bot-sunat/.venv/bin/python' +SCRIPT = '/opt/scrapers/encuentra24_empleos/scraper.py' + +default_args = { + 'owner': 'jpuma', + 'retries': 2, + 'retry_delay': timedelta(minutes=15), + 'email': ['jpuma@redneurocom.com'], + 'email_on_failure': True, + 'email_on_retry': False, + 'execution_timeout': timedelta(hours=6), +} + + +def run_scraper(**ctx): + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect( + hostname=SSH_HOST, + port=22, + username=SSH_USER, + password=SSH_PASS, + timeout=30, + ) + client.get_transport().set_keepalive(30) + + cmd = f'{VENV} {SCRIPT}' + _, stdout, stderr = client.exec_command(cmd, timeout=21600) # 6 h + + output = stdout.read().decode('utf-8', errors='replace') + errors = stderr.read().decode('utf-8', errors='replace') + client.close() + + print("=== STDOUT ===") + print(output) + if errors.strip(): + print("=== STDERR ===") + print(errors) + + if 'Finalizado:' not in output: + raise RuntimeError( + f"El scraper no completo correctamente. " + f"'Finalizado:' ausente en stdout. " + f"STDERR: {errors[:500]}" + ) + + +with DAG( + dag_id='encuentra24_empleos_daily', + default_args=default_args, + description='Scraper diario - Encuentra24 Empleos Centroamerica (6 paises)', + schedule_interval='30 5 * * *', + start_date=datetime(2024, 1, 1), + catchup=False, + max_active_runs=1, + tags=['scraping', 'encuentra24', 'ca', 'empleos'], +) as dag: + + ejecutar = PythonOperator( + task_id='ejecutar_scraper_encuentra24', + python_callable=run_scraper, + )