From b15dc9923ebfb5b5ce52d9f2b09d467351a3a80b Mon Sep 17 00:00:00 2001 From: claude Date: Wed, 29 Jul 2026 06:50:36 +0200 Subject: [PATCH] docs: README arquitectura, portales, proxy, esquemas BD --- README.md | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6d3dea9..f6de1e5 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,61 @@ # portales-hispanohablantes -Scrapers de empleos e inmuebles para portales hispanohablantes: ElEmpleo CO, Encuentra24 CA, Trabajando CL, FincaRaiz CO, Portal Inmobiliario CL, Yapo CL, Corotos DO. Airflow + ARM VPS + proxy failover fibra/dongle. \ No newline at end of file +Scrapers de empleos e inmuebles para portales hispanos. Airflow + ARM VPS 152.53.242.31. + +## Arquitectura + +- **Runtime**: Python 3 en ARM VPS (host). Venv: `/home/jpuma/proyectos/bot-sunat/.venv` +- **DAGs**: Airflow 2.9.3 (Docker). Path: `/opt/airflow/dags/` +- **Scrapers**: `/opt/scrapers//scraper.py` +- **BD**: postgres17-central (`100.75.240.87:5432`, user=`pgadmin`) +- **Proxy**: `socks5h://127.0.0.1:1090` — failover-proxy.service: fibra umacollo -> dongle Entel + +## Portales Ola 1 (operacionales) + +| Portal | DAG | Schedule | BD | Registros iniciales | +|--------|-----|----------|----|---------------------| +| ElEmpleo Colombia | `elempleo_co_daily` | 05:00 UTC | `elempleo_co` | 250+ jobs | +| FincaRaiz Colombia | `fincaraiz_co_daily` | 07:00 UTC | `fincaraiz_co` | 390+ listings | +| Portal Inmobiliario Chile | `portal_inmobiliario_cl_daily` | 07:30 UTC | `portal_inmobiliario_cl` | 630+ listings | +| Corotos Rep. Dominicana | `corotos_do_daily` | 08:30 UTC | `corotos_do` | 227+ listings | + +## Portales Ola 2 (pendiente Playwright) + +| Portal | Razon | +|--------|-------| +| Encuentra24 CA | 302 redirects, estructura SPA | +| Trabajando CL | Nuxt.js SPA | +| Yapo CL | Infinite scroll, sin paginacion HTTP | +| Fotocasa ES | Leve geo-IP | +| Inmuebles24 MX | Cloudflare WAF | +| ArgenProp AR | AWS WAF | +| Metrocuadrado CO | Imperva/Incapsula | + +## Proxy — 3 niveles + +``` +scraper -> socks5h://127.0.0.1:1090 (failover-proxy.service) + PRIMARY: :1080 -> umacollo eth0 (fibra ISP Peru) + FALLBACK: :1082 -> umacollo wwan0 (dongle Entel 20GB/mes) +Si :1090 down -> ARM directo (datacenter USA) +``` + +## Esquema BD + +### Empleos +```sql +CREATE TABLE jobs ( + job_id TEXT PRIMARY KEY, titulo TEXT, empresa TEXT, + ubicacion TEXT, url TEXT UNIQUE, fecha_pub TEXT, + categoria TEXT, scraped_at TIMESTAMP DEFAULT NOW() +); +``` + +### Inmuebles +```sql +CREATE TABLE listings ( + listing_id TEXT PRIMARY KEY, titulo TEXT, precio BIGINT, + moneda TEXT, tipo TEXT, ubicacion TEXT, url TEXT UNIQUE, + superficie_m2 DECIMAL, scraped_at TIMESTAMP DEFAULT NOW() +); +``` \ No newline at end of file