From dd3edd70884b1a3f865af67596d2b5d255222818 Mon Sep 17 00:00:00 2001 From: Grendgi Date: Thu, 4 Jun 2026 14:55:41 +0300 Subject: [PATCH] Add monitoring PF service --- .dockerignore | 23 ++ .gitea/workflows/deploy.yaml | 59 +++++ .gitignore | 10 + Dockerfile | 25 ++ README.md | 143 +++++++++++ SESSION_NOTES.md | 90 +++++++ app/__init__.py | 0 app/auth.py | 45 ++++ app/bot.py | 182 ++++++++++++++ app/config.py | 49 ++++ app/db.py | 31 +++ app/models.py | 103 ++++++++ app/scheduler.py | 57 +++++ app/scrapers/__init__.py | 5 + app/scrapers/base.py | 96 ++++++++ app/scrapers/bayut.py | 212 ++++++++++++++++ app/scrapers/propertyfinder.py | 325 +++++++++++++++++++++++++ app/services/__init__.py | 0 app/services/monitor.py | 354 +++++++++++++++++++++++++++ app/services/notifier.py | 48 ++++ app/templates/admin_login.html | 29 +++ app/templates/base.html | 88 +++++++ app/templates/employees.html | 86 +++++++ app/templates/project_detail.html | 136 +++++++++++ app/templates/project_form.html | 81 +++++++ app/templates/projects_list.html | 67 ++++++ app/templates/suggest.html | 138 +++++++++++ app/web.py | 386 ++++++++++++++++++++++++++++++ diagnose.py | 89 +++++++ docker-compose.yml | 37 +++ k8s/configmap.yaml | 11 + k8s/kustomization.yaml | 11 + k8s/namespace.yaml | 4 + k8s/secrets.yaml | 10 + k8s/server-deployment.yaml | 107 +++++++++ k8s/server-service.yaml | 18 ++ requirements.txt | 14 ++ run_bot.bat | 4 + run_scheduler.bat | 4 + run_web.bat | 4 + run_web.py | 13 + 41 files changed, 3194 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitea/workflows/deploy.yaml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 SESSION_NOTES.md create mode 100644 app/__init__.py create mode 100644 app/auth.py create mode 100644 app/bot.py create mode 100644 app/config.py create mode 100644 app/db.py create mode 100644 app/models.py create mode 100644 app/scheduler.py create mode 100644 app/scrapers/__init__.py create mode 100644 app/scrapers/base.py create mode 100644 app/scrapers/bayut.py create mode 100644 app/scrapers/propertyfinder.py create mode 100644 app/services/__init__.py create mode 100644 app/services/monitor.py create mode 100644 app/services/notifier.py create mode 100644 app/templates/admin_login.html create mode 100644 app/templates/base.html create mode 100644 app/templates/employees.html create mode 100644 app/templates/project_detail.html create mode 100644 app/templates/project_form.html create mode 100644 app/templates/projects_list.html create mode 100644 app/templates/suggest.html create mode 100644 app/web.py create mode 100644 diagnose.py create mode 100644 docker-compose.yml create mode 100644 k8s/configmap.yaml create mode 100644 k8s/kustomization.yaml create mode 100644 k8s/namespace.yaml create mode 100644 k8s/secrets.yaml create mode 100644 k8s/server-deployment.yaml create mode 100644 k8s/server-service.yaml create mode 100644 requirements.txt create mode 100644 run_bot.bat create mode 100644 run_scheduler.bat create mode 100644 run_web.bat create mode 100644 run_web.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e03c9c9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,23 @@ +.venv/ +__pycache__/ +**/__pycache__/ +*.pyc +*.pyo +.git/ +.gitignore +.idea/ +.vscode/ +.claude/ +.DS_Store +data/ +*.db +*.db-journal +*.log +*.bat +diagnose.py +README.md +SESSION_NOTES.md +Dockerfile +docker-compose.yml +.dockerignore +.env diff --git a/.gitea/workflows/deploy.yaml b/.gitea/workflows/deploy.yaml new file mode 100644 index 0000000..ef7c1eb --- /dev/null +++ b/.gitea/workflows/deploy.yaml @@ -0,0 +1,59 @@ +name: Build and Deploy + +on: + push: + branches: [main] + +env: + INTERNAL_REGISTRY: gitea-http.gitea.svc.cluster.local:3000 + NODE_REGISTRY: localhost:30300 + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Docker CLI + run: | + curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-27.5.1.tgz \ + | tar xz --strip-components=1 -C /usr/local/bin docker/docker + docker version + + - name: Install kubectl + run: | + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl + mv kubectl /usr/local/bin/ + kubectl version --client + + - name: Login to Gitea Registry + run: | + echo "${{ secrets.REGISTRY_PASSWORD }}" | \ + docker login ${{ env.INTERNAL_REGISTRY }} \ + -u ${{ secrets.REGISTRY_USERNAME }} --password-stdin + + - name: Build and push server + run: | + docker build -f Dockerfile \ + -t ${{ env.INTERNAL_REGISTRY }}/admin/monitoring-pf-server:${{ github.sha }} \ + -t ${{ env.INTERNAL_REGISTRY }}/admin/monitoring-pf-server:latest \ + . + docker push ${{ env.INTERNAL_REGISTRY }}/admin/monitoring-pf-server:${{ github.sha }} + docker push ${{ env.INTERNAL_REGISTRY }}/admin/monitoring-pf-server:latest + + - name: Deploy to Kubernetes + env: + KUBECONFIG: /kubeconfig/config + run: | + kubectl apply -f k8s/namespace.yaml + kubectl apply -f k8s/secrets.yaml + kubectl apply -f k8s/configmap.yaml + kubectl apply -f k8s/server-deployment.yaml + kubectl apply -f k8s/server-service.yaml + kubectl -n monitoring-pf set image deployment/monitoring-pf-server \ + web=${{ env.NODE_REGISTRY }}/admin/monitoring-pf-server:${{ github.sha }} \ + bot=${{ env.NODE_REGISTRY }}/admin/monitoring-pf-server:${{ github.sha }} \ + scheduler=${{ env.NODE_REGISTRY }}/admin/monitoring-pf-server:${{ github.sha }} + kubectl -n monitoring-pf rollout status deployment/monitoring-pf-server --timeout=180s diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..79bdd87 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +.venv/ +__pycache__/ +*.pyc +.env +data/ +*.db +*.db-journal +.idea/ +.vscode/ +*.log diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0077163 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.12-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends tini \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt ./ +RUN pip install -r requirements.txt + +COPY app ./app +COPY run_web.py ./ + +RUN mkdir -p /app/data + +EXPOSE 8000 + +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["python", "run_web.py"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..88cc33b --- /dev/null +++ b/README.md @@ -0,0 +1,143 @@ +# DLD Monitor + +Внутренний инструмент для агентства недвижимости в Дубае: мониторит цены объявлений конкурентов на **PropertyFinder.ae** и **Bayut.com** по DLD Permit Number, шлёт уведомления в Telegram при: + +- 📈📉 изменении цены конкурента, +- ❌ удалении объявления (404 / withdrawn), +- 🆕 появлении нового объявления с тем же permit (новый брокер выставил ту же квартиру). + +## Архитектура + +``` +┌──────────────┐ ┌──────────────────┐ ┌─────────────────┐ +│ Web UI │ │ Scheduler │ │ Telegram Bot │ +│ (FastAPI) │ │ (APScheduler) │ │ (polling) │ +│ add project │ │ every N hours │ │ /start /check │ +└──────┬───────┘ └────────┬─────────┘ └────────┬────────┘ + │ │ │ + └──────────────┬──────┴───────────────────────┘ + ▼ + ┌────────────────┐ + │ monitor │ + │ service │ ← скрапит PF и Bayut, пишет в SQLite + └────────────────┘ + │ + ▼ уведомления в TG конкретному employee +``` + +## Локальный запуск (Windows) + +### 1. Создать виртуальное окружение и поставить зависимости + +```powershell +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +pip install -r requirements.txt +``` + +### 2. Получить токен Telegram-бота + +Уже есть? Отлично. Если нет: +1. Откройте Telegram, найдите [@BotFather](https://t.me/BotFather). +2. Отправьте `/newbot`, придумайте имя. +3. Скопируйте токен вида `123456:ABC-DEF…`. + +### 3. Создать `.env` + +```powershell +Copy-Item .env.example .env +notepad .env +``` + +В `.env` вставьте: + +``` +TG_BOT_TOKEN=ваш_токен_от_botfather +SCRAPE_INTERVAL_HOURS=4 +ADMIN_CHAT_ID= # опционально — куда слать системные ошибки +``` + +### 4. Запустить три процесса (в **трёх** разных окнах PowerShell) + +**Окно 1 — веб-интерфейс:** +```powershell +.\.venv\Scripts\Activate.ps1 +python run_web.py +``` +Откройте http://127.0.0.1:8000 + +**Окно 2 — Telegram-бот:** +```powershell +.\.venv\Scripts\Activate.ps1 +python -m app.bot +``` + +**Окно 3 — фоновый сканер:** +```powershell +.\.venv\Scripts\Activate.ps1 +python -m app.scheduler +``` + +## Первое использование + +1. Откройте бота в Telegram и отправьте `/start` — он зарегистрирует ваш `chat_id`. +2. В веб-UI перейдите в **Сотрудники** — убедитесь, что вы там есть с ✓ TG. +3. Нажмите **+ Новый проект**, заполните: + - Название (например: «Marina Pinnacle 1502, 2BR») + - DLD Permit Number (Trakheesi) + - Тип сделки (продажа/аренда) + - Владелец = вы +4. На странице проекта нажмите **Проверить сейчас** — система найдёт все объявления конкурентов с этим permit на PF и Bayut. +5. Дальше фоновый сканер сам будет проверять каждые `SCRAPE_INTERVAL_HOURS` часов и слать уведомления в Telegram. + +## Команды бота + +- `/start` — подключить себя как сотрудника (запоминает chat_id) +- `/list` — список ваших проектов +- `/check` — запустить проверку всех ваших проектов сейчас +- `/whoami` — показать свой chat_id + +## Структура + +``` +app/ +├── config.py настройки из .env +├── db.py SQLAlchemy engine + session +├── models.py Employee, Project, CompetitorListing, PriceHistory +├── web.py FastAPI роуты и UI +├── bot.py Telegram-бот +├── scheduler.py APScheduler фоновый сканер +├── scrapers/ +│ ├── base.py httpx + парсинг __NEXT_DATA__ +│ ├── propertyfinder.py +│ └── bayut.py +├── services/ +│ ├── monitor.py детект изменений, основная бизнес-логика +│ └── notifier.py отправка в TG +└── templates/ Jinja2 (Bootstrap 5) +data/ +└── monitor.db SQLite (создаётся автоматически) +``` + +## Перенос на сервер + +Когда придёт время — нужно: +1. Поставить Python 3.11+ на сервер (Linux). +2. Скопировать репозиторий, `pip install -r requirements.txt`. +3. Создать `.env`. +4. Поставить три процесса под `systemd`: + - `dld-monitor-web.service` → `python run_web.py` + - `dld-monitor-bot.service` → `python -m app.bot` + - `dld-monitor-scheduler.service` → `python -m app.scheduler` +5. Поставить nginx + TLS перед веб-портом (8000). + +## Возможные проблемы скрапинга + +PF/Bayut могут начать блокировать запросы при частых обращениях. Признаки: +- В логах сканера видны `Blocked by site (403/429)`. +- Поиск возвращает 0 объявлений, хотя они должны быть. + +Что делать: +1. Уменьшите `SCRAPE_INTERVAL_HOURS` (реже = меньше риска). +2. Если не помогает — добавьте Playwright (headless-браузер). Заготовка: `pip install playwright && playwright install chromium`, затем заменить `fetch_html` на запуск через Playwright. +3. Опционально — прокси. diff --git a/SESSION_NOTES.md b/SESSION_NOTES.md new file mode 100644 index 0000000..5564646 --- /dev/null +++ b/SESSION_NOTES.md @@ -0,0 +1,90 @@ +# Журнал сессии (2026-05-25) + +## Что построили + +Внутренний мониторинг цен конкурентов на propertyfinder.ae и bayut.com для HOME LIGA REAL ESTATE. + +**Стек:** FastAPI + Jinja2 (Bootstrap 5) + SQLAlchemy/SQLite + APScheduler + python-telegram-bot + httpx/BS4. + +**Три процесса** (каждый — свой .bat-лаунчер): +- `run_web.bat` — веб-UI на http://127.0.0.1:8000 +- `run_bot.bat` — Telegram-бот (polling) +- `run_scheduler.bat` — фоновый сканер каждые `SCRAPE_INTERVAL_HOURS` часов + +## Эволюция архитектуры + +### Первая попытка (отвергнута) +**Идея:** сотрудник вводит DLD Permit Number своего объекта → система автоматически ищет «то же permit» на PF и Bayut → находит объявления конкурентов. + +**Почему не сработало:** +1. В Дубае **каждый брокер получает свой permit на свою публикацию**. Два брокера, рекламирующих одну квартиру = два разных permit. Permit не идентифицирует физический объект — он идентифицирует конкретное объявление конкретного брокера. +2. PF на странице объявления показывает permit **картинкой** через сервис верификации, не plain text (anti-scraping). +3. PF search `?q=` — free-text по названию/описанию, не структурированный фильтр. + +### Финальная архитектура +**Manual URL list + опциональные подсказки:** +1. Сотрудник создаёт проект (название, тип сделки, владелец, опц. building/bedrooms/sqft). +2. На странице проекта **вручную вставляет URL** объявления конкурента → система делает single-page fetch, парсит `__NEXT_DATA__`, добавляет в трекинг. +3. Если указано здание — кнопка «🔍 Подобрать похожие» ищет на PF/Bayut по `building + bedrooms` и предлагает кандидатов с кнопкой «+ Отслеживать». +4. Каждые 4 часа фоновый сканер делает refetch каждого отслеживаемого URL → детектит: + - 📈📉 изменение цены + - ❌ удаление (URL отдаёт 404) + - ♻️ возвращение из удалённого статуса +5. Уведомления — в Telegram личкой владельцу проекта. + +## Модель данных + +- `Employee` — name, tg_chat_id (опц.), tg_username +- `Project` — title, deal_type, owner_id, our_price, building, bedrooms, size_sqft, our_url, dld_permit (все после `owner` — опционально) +- `CompetitorListing` — source (PF|Bayut), external_id, url, current_price, status (active|removed), agent_name, agency_name, first_seen, last_seen +- `PriceHistory` — listing_id, price, recorded_at + +## Как подключиться сотруднику + +1. В TG найти бота → отправить `/start`. +2. Отправить `/whoami` → бот пришлёт chat_id. +3. В вебе http://127.0.0.1:8000/employees → найти/создать запись → вставить chat_id → Сохранить. + +## Известные ограничения + +- **PF/Bayut могут блокировать** при частых запросах (видно как `Blocked by site (403/429)` в логах). Решение — увеличить интервал; если уже не помогает — добавить Playwright fallback. +- **Подсказки эвристические**: ищем по совпадению building name в title + bedrooms-фильтр. Могут попасть «другие квартиры в том же здании» — поэтому добавление в трекинг через ручное подтверждение. +- **Permit как plain text** на PF не отдаётся. Если когда-нибудь понадобится — нужен OCR на verification-image. + +## Что в .env + +``` +TG_BOT_TOKEN=<токен от @BotFather> +SCRAPE_INTERVAL_HOURS=4 +ADMIN_CHAT_ID= # опц. +``` + +## Структура проекта + +``` +DLD Permit Number/ +├── run_web.bat, run_bot.bat, run_scheduler.bat +├── run_web.py +├── requirements.txt +├── .env (не в git — содержит токен) +├── data/monitor.db (создаётся автоматически) +├── app/ +│ ├── config.py ← settings + резолвит относительные SQLite-пути в абсолютные +│ ├── db.py, models.py +│ ├── web.py ← FastAPI: CRUD проектов, /listings add/delete, /suggest +│ ├── bot.py ← /start, /whoami, /list, /check +│ ├── scheduler.py +│ ├── scrapers/{base,propertyfinder,bayut}.py +│ ├── services/{monitor,notifier}.py +│ └── templates/ ← projects_list, project_form, project_detail, suggest, employees, base +``` + +## Что протестировать в первую очередь + +1. Удалить старый `data/monitor.db` (схема изменилась). +2. Запустить три .bat файла. +3. `/start` боту → `/whoami` → chat_id → вписать в Сотрудники. +4. Создать тестовый проект (Aykon City Tower B, 2BR). +5. Вставить URL реального объявления конкурента → проверить, что добавилось с ценой/брокером. +6. Жмякнуть «Подобрать похожие» → посмотреть кандидатов. +7. Жмякнуть «Проверить сейчас» → если цена не менялась, изменений не будет (это нормально); для теста алертов можно поменять цену в БД руками или подождать реального изменения. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/auth.py b/app/auth.py new file mode 100644 index 0000000..4734d98 --- /dev/null +++ b/app/auth.py @@ -0,0 +1,45 @@ +"""Lightweight admin gate for the web UI. + +A single shared PIN (`ADMIN_PIN` in .env) unlocks destructive actions +(deleting projects/competitors/employees, editing employees) for the browser +session via an HMAC-signed cookie. No per-user accounts — this is an internal +localhost tool, not a public app. + +If `ADMIN_PIN` is empty the gate is OPEN (nothing restricted) so the tool is +never bricked by a missing setting; set a PIN to actually restrict. +""" +from __future__ import annotations + +import hashlib +import hmac + +from starlette.requests import Request + +from app.config import settings + +ADMIN_COOKIE = "dld_admin" +COOKIE_MAX_AGE = 60 * 60 * 8 # 8 hours + + +def admin_configured() -> bool: + return bool(settings.admin_pin) + + +def admin_token() -> str | None: + """The cookie value that proves admin: HMAC(pin, marker). None if no PIN.""" + if not settings.admin_pin: + return None + return hmac.new(settings.admin_pin.encode(), b"dld-admin-v1", hashlib.sha256).hexdigest() + + +def pin_ok(pin: str) -> bool: + """Constant-time check of a submitted PIN against the configured one.""" + return bool(settings.admin_pin) and hmac.compare_digest((pin or "").strip(), settings.admin_pin) + + +def request_is_admin(request: Request) -> bool: + if not settings.admin_pin: + return True # gate not configured → open + token = request.cookies.get(ADMIN_COOKIE) + expected = admin_token() + return bool(token and expected and hmac.compare_digest(token, expected)) diff --git a/app/bot.py b/app/bot.py new file mode 100644 index 0000000..8c0ac05 --- /dev/null +++ b/app/bot.py @@ -0,0 +1,182 @@ +"""Telegram bot — registers employees by chat_id and lets them trigger checks. + +Run as a separate process: `python -m app.bot`. + +Bot commands (set via @BotFather → /setcommands): + start - Подключить себя как сотрудника + list - Список своих проектов + check - Проверить все мои проекты сейчас + whoami - Показать свой chat_id +""" + +from __future__ import annotations + +import asyncio +import logging + +from sqlalchemy.orm import joinedload +from telegram import Update +from telegram.ext import ( + Application, + CommandHandler, + ContextTypes, +) + +from app.config import settings +from app.db import SessionLocal, init_db +from app.models import Employee, Project +from app.services.monitor import run_check_for_project + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +logger = logging.getLogger(__name__) + + +async def cmd_start(update: Update, _: ContextTypes.DEFAULT_TYPE) -> None: + if not update.effective_user or not update.effective_chat: + return + user = update.effective_user + chat_id = str(update.effective_chat.id) + username = user.username + + db = SessionLocal() + try: + existing = ( + db.query(Employee).filter(Employee.tg_chat_id == chat_id).first() + ) + if existing: + await update.message.reply_text( + f"✅ Вы уже подключены как {existing.name}.\n" + f"chat_id: {chat_id}", + parse_mode="HTML", + ) + return + + # Try to find by username (admin pre-created employee w/o chat_id) + if username: + placeholder = ( + db.query(Employee) + .filter(Employee.tg_username == username, Employee.tg_chat_id.is_(None)) + .first() + ) + if placeholder: + placeholder.tg_chat_id = chat_id + db.commit() + await update.message.reply_text( + f"✅ Привет, {placeholder.name}! Вы успешно подключены.\n" + f"Уведомления будут приходить сюда.", + parse_mode="HTML", + ) + return + + # Create a new employee record from this user + name = (user.full_name or username or f"user_{chat_id}").strip() + e = Employee(name=name, tg_chat_id=chat_id, tg_username=username) + db.add(e) + db.commit() + await update.message.reply_text( + f"👋 Привет, {name}! Вы зарегистрированы как сотрудник.\n" + f"Откройте веб-интерфейс и создайте проекты, чтобы получать уведомления.\n" + f"chat_id: {chat_id}", + parse_mode="HTML", + ) + finally: + db.close() + + +async def cmd_whoami(update: Update, _: ContextTypes.DEFAULT_TYPE) -> None: + if not update.effective_chat: + return + chat_id = str(update.effective_chat.id) + db = SessionLocal() + try: + e = db.query(Employee).filter(Employee.tg_chat_id == chat_id).first() + if e: + await update.message.reply_text( + f"Вы: {e.name}\nchat_id: {chat_id}", + parse_mode="HTML", + ) + else: + await update.message.reply_text( + f"Вы пока не подключены. Отправьте /start.\nchat_id: {chat_id}", + parse_mode="HTML", + ) + finally: + db.close() + + +async def cmd_list(update: Update, _: ContextTypes.DEFAULT_TYPE) -> None: + if not update.effective_chat: + return + chat_id = str(update.effective_chat.id) + db = SessionLocal() + try: + e = ( + db.query(Employee) + .options(joinedload(Employee.projects)) + .filter(Employee.tg_chat_id == chat_id) + .first() + ) + if not e: + await update.message.reply_text("Сначала /start.") + return + if not e.projects: + await update.message.reply_text("У вас пока нет проектов.") + return + lines = [f"Ваши проекты ({len(e.projects)}):"] + for p in e.projects: + lines.append( + f"• #{p.id} {p.title} — {p.dld_permit} " + f"({p.deal_type.value})" + ) + await update.message.reply_text("\n".join(lines), parse_mode="HTML") + finally: + db.close() + + +async def cmd_check(update: Update, _: ContextTypes.DEFAULT_TYPE) -> None: + if not update.effective_chat: + return + chat_id = str(update.effective_chat.id) + db = SessionLocal() + try: + e = ( + db.query(Employee) + .options(joinedload(Employee.projects)) + .filter(Employee.tg_chat_id == chat_id) + .first() + ) + if not e: + await update.message.reply_text("Сначала /start.") + return + if not e.projects: + await update.message.reply_text("У вас нет проектов.") + return + ids = [p.id for p in e.projects] + finally: + db.close() + + await update.message.reply_text(f"⏳ Запускаю проверку {len(ids)} проектов…") + total_changes = 0 + for pid in ids: + try: + total_changes += await asyncio.to_thread(run_check_for_project, pid) + except Exception as ex: + logger.exception("check failed for %s: %s", pid, ex) + await update.message.reply_text(f"✅ Готово. Изменений: {total_changes}") + + +def main() -> None: + if not settings.tg_bot_token: + raise SystemExit("TG_BOT_TOKEN не задан в .env") + init_db() + app = Application.builder().token(settings.tg_bot_token).build() + app.add_handler(CommandHandler("start", cmd_start)) + app.add_handler(CommandHandler("whoami", cmd_whoami)) + app.add_handler(CommandHandler("list", cmd_list)) + app.add_handler(CommandHandler("check", cmd_check)) + logger.info("Bot polling…") + app.run_polling(allowed_updates=Update.ALL_TYPES) + + +if __name__ == "__main__": + main() diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..5ccab5a --- /dev/null +++ b/app/config.py @@ -0,0 +1,49 @@ +from pathlib import Path + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +BASE_DIR = Path(__file__).resolve().parent.parent +DATA_DIR = BASE_DIR / "data" +DATA_DIR.mkdir(exist_ok=True) + + +def _resolve_sqlite_url(url: str) -> str: + """SQLAlchemy SQLite URL with a relative path is interpreted against CWD, + which breaks when the app is started from any directory other than the + project root. Anchor relative SQLite paths to BASE_DIR.""" + prefix = "sqlite:///" + if not url.startswith(prefix): + return url + path_part = url[len(prefix):] + p = Path(path_part) + if p.is_absolute(): + return url + resolved = (BASE_DIR / p).resolve() + resolved.parent.mkdir(parents=True, exist_ok=True) + return f"{prefix}{resolved}" + + +class Settings(BaseSettings): + model_config = SettingsConfigDict( + env_file=BASE_DIR / ".env", + env_file_encoding="utf-8", + extra="ignore", + ) + + tg_bot_token: str = "" + web_host: str = "127.0.0.1" + web_port: int = 8000 + public_base_path: str = "" + scrape_interval_hours: int = 4 + database_url: str = f"sqlite:///{DATA_DIR / 'monitor.db'}" + admin_chat_id: str = "" + # Shared PIN that unlocks destructive web actions (delete/edit). Empty = gate + # disabled (everything allowed). See app/auth.py. + admin_pin: str = "" + + def model_post_init(self, __context) -> None: + self.database_url = _resolve_sqlite_url(self.database_url) + + +settings = Settings() diff --git a/app/db.py b/app/db.py new file mode 100644 index 0000000..6bd9a88 --- /dev/null +++ b/app/db.py @@ -0,0 +1,31 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import DeclarativeBase, sessionmaker + +from app.config import settings + + +engine = create_engine( + settings.database_url, + connect_args={"check_same_thread": False} if settings.database_url.startswith("sqlite") else {}, + future=True, +) + +SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, future=True) + + +class Base(DeclarativeBase): + pass + + +def get_db(): + db = SessionLocal() + try: + yield db + finally: + db.close() + + +def init_db(): + from app import models # noqa: F401 — registers models on Base + + Base.metadata.create_all(bind=engine) diff --git a/app/models.py b/app/models.py new file mode 100644 index 0000000..d7bec9b --- /dev/null +++ b/app/models.py @@ -0,0 +1,103 @@ +from datetime import datetime +from enum import Enum + +from sqlalchemy import DateTime, Enum as SAEnum, Float, ForeignKey, Integer, String, Text, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db import Base + + +class DealType(str, Enum): + SALE = "sale" + RENT = "rent" + + +class Source(str, Enum): + PROPERTYFINDER = "propertyfinder" + BAYUT = "bayut" + + +class ListingStatus(str, Enum): + ACTIVE = "active" + REMOVED = "removed" + + +class Employee(Base): + __tablename__ = "employees" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + name: Mapped[str] = mapped_column(String(200)) + tg_chat_id: Mapped[str | None] = mapped_column(String(64), unique=True, nullable=True) + tg_username: Mapped[str | None] = mapped_column(String(200), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + + projects: Mapped[list["Project"]] = relationship(back_populates="owner") + + +class Project(Base): + """Наш проект — квартира, которую агентство рекламирует.""" + + __tablename__ = "projects" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + title: Mapped[str] = mapped_column(String(300)) + deal_type: Mapped[DealType] = mapped_column(SAEnum(DealType)) + our_price: Mapped[float | None] = mapped_column(Float, nullable=True) + notes: Mapped[str | None] = mapped_column(Text, nullable=True) + + # Опциональные параметры — используются для подсказок похожих объявлений + dld_permit: Mapped[str | None] = mapped_column(String(100), index=True, nullable=True) + building: Mapped[str | None] = mapped_column(String(300), nullable=True) + bedrooms: Mapped[int | None] = mapped_column(Integer, nullable=True) + size_sqft: Mapped[float | None] = mapped_column(Float, nullable=True) + our_url: Mapped[str | None] = mapped_column(Text, nullable=True) + + owner_id: Mapped[int] = mapped_column(ForeignKey("employees.id")) + owner: Mapped[Employee] = relationship(back_populates="projects") + + created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + last_checked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + + listings: Mapped[list["CompetitorListing"]] = relationship( + back_populates="project", cascade="all, delete-orphan" + ) + + +class CompetitorListing(Base): + """Объявление конкурента, найденное на PF/Bayut по DLD permit нашего проекта.""" + + __tablename__ = "competitor_listings" + __table_args__ = (UniqueConstraint("project_id", "source", "external_id", name="uq_listing"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + project_id: Mapped[int] = mapped_column(ForeignKey("projects.id")) + project: Mapped[Project] = relationship(back_populates="listings") + + source: Mapped[Source] = mapped_column(SAEnum(Source)) + external_id: Mapped[str] = mapped_column(String(100)) # ID на стороне PF/Bayut + url: Mapped[str] = mapped_column(Text) + title: Mapped[str | None] = mapped_column(String(500), nullable=True) + agent_name: Mapped[str | None] = mapped_column(String(300), nullable=True) + agency_name: Mapped[str | None] = mapped_column(String(300), nullable=True) + + current_price: Mapped[float | None] = mapped_column(Float, nullable=True) + currency: Mapped[str | None] = mapped_column(String(10), nullable=True, default="AED") + status: Mapped[ListingStatus] = mapped_column(SAEnum(ListingStatus), default=ListingStatus.ACTIVE) + + first_seen_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + last_seen_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + + price_history: Mapped[list["PriceHistory"]] = relationship( + back_populates="listing", cascade="all, delete-orphan", order_by="PriceHistory.recorded_at.desc()" + ) + + +class PriceHistory(Base): + __tablename__ = "price_history" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + listing_id: Mapped[int] = mapped_column(ForeignKey("competitor_listings.id")) + listing: Mapped[CompetitorListing] = relationship(back_populates="price_history") + + price: Mapped[float | None] = mapped_column(Float, nullable=True) + recorded_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) diff --git a/app/scheduler.py b/app/scheduler.py new file mode 100644 index 0000000..79a20cb --- /dev/null +++ b/app/scheduler.py @@ -0,0 +1,57 @@ +"""Background scheduler — runs run_check_all() every N hours. + +Run as a separate process: `python -m app.scheduler`. +""" + +from __future__ import annotations + +import logging +import time + +from apscheduler.schedulers.blocking import BlockingScheduler +from apscheduler.triggers.interval import IntervalTrigger + +from app.config import settings +from app.db import init_db +from app.services.monitor import run_check_all + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +logger = logging.getLogger(__name__) + + +def job() -> None: + logger.info("Scheduled scan starting…") + start = time.time() + summary = run_check_all() + elapsed = time.time() - start + total_changes = sum(c for c in summary.values() if c > 0) + logger.info( + "Scan done in %.1fs. Projects: %d, total changes: %d", + elapsed, len(summary), total_changes, + ) + + +def main() -> None: + init_db() + hours = max(1, settings.scrape_interval_hours) + scheduler = BlockingScheduler(timezone="UTC") + scheduler.add_job( + job, + trigger=IntervalTrigger(hours=hours), + # Omit next_run_time so APScheduler defaults the first run to now+interval + # (i.e. don't fire immediately at startup, fire after one interval, then + # every interval). Passing next_run_time=None instead creates the job in a + # PAUSED state and it never fires — that was the bug. + id="periodic-scan", + max_instances=1, + coalesce=True, + ) + logger.info("Scheduler started — interval %d hour(s).", hours) + try: + scheduler.start() + except (KeyboardInterrupt, SystemExit): + logger.info("Scheduler stopped.") + + +if __name__ == "__main__": + main() diff --git a/app/scrapers/__init__.py b/app/scrapers/__init__.py new file mode 100644 index 0000000..608fec5 --- /dev/null +++ b/app/scrapers/__init__.py @@ -0,0 +1,5 @@ +from app.scrapers.base import ScrapedListing +from app.scrapers.bayut import BayutScraper +from app.scrapers.propertyfinder import PropertyFinderScraper + +__all__ = ["ScrapedListing", "BayutScraper", "PropertyFinderScraper"] diff --git a/app/scrapers/base.py b/app/scrapers/base.py new file mode 100644 index 0000000..dab18b6 --- /dev/null +++ b/app/scrapers/base.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import json +import logging +import re +from dataclasses import dataclass + +import httpx +from bs4 import BeautifulSoup + +logger = logging.getLogger(__name__) + +DEFAULT_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/131.0.0.0 Safari/537.36" + ), + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "gzip, deflate, br", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", +} + + +@dataclass +class ScrapedListing: + source: str # "propertyfinder" | "bayut" + external_id: str # listing id on the source + url: str + title: str | None + price: float | None + currency: str | None + permit_number: str | None + agent_name: str | None + agency_name: str | None + is_active: bool = True + + +class ScraperError(Exception): + pass + + +def fetch_html(url: str, timeout: float = 30.0) -> str: + """GET a URL with browser-like headers. Raises ScraperError on non-2xx.""" + try: + with httpx.Client(headers=DEFAULT_HEADERS, follow_redirects=True, timeout=timeout) as client: + r = client.get(url) + if r.status_code in (403, 429): + raise ScraperError(f"Blocked by site ({r.status_code}) at {url}") + if r.status_code == 404: + return "" + r.raise_for_status() + return r.text + except httpx.HTTPError as e: + raise ScraperError(f"HTTP error for {url}: {e}") from e + + +_NEXT_DATA_RE = re.compile( + r']+id="__NEXT_DATA__"[^>]*>(.*?)', + re.DOTALL, +) + + +def extract_next_data(html: str) -> dict | None: + """Extract Next.js __NEXT_DATA__ JSON blob — both PF and Bayut are Next.js apps.""" + if not html: + return None + m = _NEXT_DATA_RE.search(html) + if not m: + # Fallback via BeautifulSoup if regex misses (rare). + soup = BeautifulSoup(html, "lxml") + tag = soup.find("script", id="__NEXT_DATA__") + if not tag or not tag.string: + return None + raw = tag.string + else: + raw = m.group(1) + try: + return json.loads(raw) + except json.JSONDecodeError as e: + logger.warning("Failed to parse __NEXT_DATA__: %s", e) + return None + + +def parse_price(value) -> float | None: + if value is None: + return None + if isinstance(value, (int, float)): + return float(value) + s = re.sub(r"[^\d.]", "", str(value)) + try: + return float(s) if s else None + except ValueError: + return None diff --git a/app/scrapers/bayut.py b/app/scrapers/bayut.py new file mode 100644 index 0000000..36bfc3d --- /dev/null +++ b/app/scrapers/bayut.py @@ -0,0 +1,212 @@ +"""Bayut.com scraper. + +Two operations: +- fetch_listing(url): read a listing detail page → ScrapedListing. +- search_similar(building, bedrooms, deal_type): search Bayut for similar candidates. + +Bayut is a Next.js app; __NEXT_DATA__ contains the property in pageProps. +Unlike PF, Bayut shows the permit number as text in the JSON. +""" + +from __future__ import annotations + +import logging +import re +from urllib.parse import quote_plus, urljoin + +from app.scrapers.base import ( + ScrapedListing, + ScraperError, + extract_next_data, + fetch_html, + parse_price, +) + +logger = logging.getLogger(__name__) + +BASE_URL = "https://www.bayut.com" +SOURCE = "bayut" + + +def _path_for_deal(deal_type: str) -> str: + return "to-buy" if deal_type == "sale" else "to-rent" + + +def _walk(node): + if isinstance(node, dict): + yield node + for v in node.values(): + yield from _walk(v) + elif isinstance(node, list): + for it in node: + yield from _walk(it) + + +def _extract_price(item: dict) -> tuple[float | None, str | None]: + price = item.get("price") + if isinstance(price, dict): + val = price.get("value") or price.get("amount") + cur = price.get("currency") or "AED" + return parse_price(val), cur + if isinstance(price, (int, float, str)): + return parse_price(price), "AED" + return None, "AED" + + +def _extract_broker(item: dict) -> tuple[str | None, str | None]: + agency = item.get("agency") or {} + agency_name = agency.get("name") if isinstance(agency, dict) else None + agent_name = item.get("contactName") or item.get("agentName") or item.get("ownerAgent", {}).get("name") if isinstance(item.get("ownerAgent"), dict) else item.get("contactName") + return agent_name, agency_name + + +def _extract_permit(item: dict) -> str | None: + for key in ("permitNumber", "permit_number", "rera", "trakheesi", "permit"): + v = item.get(key) + if v: + return str(v).strip() + return None + + +_ID_FROM_URL = re.compile(r"details-(\d+)\.html(?:[?#].*)?$") + + +def _extract_id_from_url(url: str) -> str | None: + m = _ID_FROM_URL.search(url) + return m.group(1) if m else None + + +def _is_listing_dict(item: dict) -> bool: + if not isinstance(item, dict): + return False + has_price = "price" in item + has_id = any(k in item for k in ("externalID", "id", "objectID")) + return has_price and has_id + + +class BayutScraper: + source = SOURCE + + def fetch_listing(self, url: str) -> ScrapedListing | None: + try: + html = fetch_html(url) + except ScraperError as e: + logger.warning("Bayut refetch failed for %s: %s", url, e) + return None + + if not html: + return ScrapedListing( + source=SOURCE, external_id=_extract_id_from_url(url) or "", url=url, + title=None, price=None, currency=None, permit_number=None, + agent_name=None, agency_name=None, is_active=False, + ) + + data = extract_next_data(html) + if not data: + return None + + best = None + best_score = -1 + for node in _walk(data): + if not _is_listing_dict(node): + continue + score = 0 + if "title" in node or "name" in node: + score += 2 + if "agency" in node or "contactName" in node: + score += 2 + if "rooms" in node or "bedrooms" in node: + score += 1 + if score > best_score: + best_score = score + best = node + + if best is None: + logger.warning("Bayut: no listing dict found in __NEXT_DATA__ for %s", url) + return None + + price, currency = _extract_price(best) + agent_name, agency_name = _extract_broker(best) + ext_id = ( + str(best.get("externalID") or best.get("id") or "") + or _extract_id_from_url(url) + or "" + ) + return ScrapedListing( + source=SOURCE, + external_id=ext_id, + url=url, + title=best.get("title") or best.get("name"), + price=price, + currency=currency, + permit_number=_extract_permit(best), + agent_name=agent_name, + agency_name=agency_name, + is_active=True, + ) + + def search_similar( + self, + building: str | None, + bedrooms: int | None, + deal_type: str, + limit: int = 20, + location_url: str | None = None, + ) -> list[ScrapedListing]: + if not building: + return [] + path = _path_for_deal(deal_type) + q = quote_plus(building.strip()) + url = f"{BASE_URL}/{path}/property/dubai/?q={q}" + if bedrooms is not None: + url += f"&beds_in={bedrooms}" + logger.info("Bayut search_similar: %s", url) + + try: + html = fetch_html(url) + except ScraperError as e: + logger.warning("Bayut search failed: %s", e) + return [] + + data = extract_next_data(html) + if not data: + return [] + + results: list[ScrapedListing] = [] + seen_ids: set[str] = set() + for node in _walk(data): + if not _is_listing_dict(node): + continue + ext_id = str(node.get("externalID") or node.get("id") or "") + if not ext_id or ext_id in seen_ids: + continue + + title = node.get("title") or node.get("name") or "" + if building.lower() not in (title or "").lower(): + slug = str(node.get("slug") or "").lower() + building_token = building.lower().replace(" ", "-") + if building_token not in slug: + continue + + seen_ids.add(ext_id) + price, currency = _extract_price(node) + agent_name, agency_name = _extract_broker(node) + cand_url = urljoin(BASE_URL, f"/property/details-{ext_id}.html") + + results.append( + ScrapedListing( + source=SOURCE, + external_id=ext_id, + url=cand_url, + title=title or None, + price=price, + currency=currency, + permit_number=_extract_permit(node), + agent_name=agent_name, + agency_name=agency_name, + is_active=True, + ) + ) + if len(results) >= limit: + break + return results diff --git a/app/scrapers/propertyfinder.py b/app/scrapers/propertyfinder.py new file mode 100644 index 0000000..9eca80d --- /dev/null +++ b/app/scrapers/propertyfinder.py @@ -0,0 +1,325 @@ +"""PropertyFinder.ae scraper. + +Two operations: +- fetch_listing(url): read a listing detail page → ScrapedListing (title/price/agent/permit). +- search_similar(building, bedrooms, deal_type): search PF for similar candidates + by building name + bedrooms filter → list[ScrapedListing]. + +PF is a Next.js app — listing data sits in +{% block scripts %}{% endblock %} + + diff --git a/app/templates/employees.html b/app/templates/employees.html new file mode 100644 index 0000000..15e6d3e --- /dev/null +++ b/app/templates/employees.html @@ -0,0 +1,86 @@ +{% extends "base.html" %} +{% block title %}Сотрудники — DLD Monitor{% endblock %} +{% block content %} + +

Сотрудники

+ +
+ Как подключить Telegram: +
    +
  1. Сотрудник пишет боту в Telegram команду /whoami — бот вернёт его chat_id.
    2. +
  2. Вставьте этот chat_id в поле справа от имени и нажмите «Сохранить».
    3. +
+ Если сотрудник пока не знает свой chat_id — пусть просто отправит /start, бот ответит и пришлёт id. +
+ +
+
+ +
+
+ +
+
+ +
+
+ +{% if not request.state.is_admin %} +
+ 🔒 Редактирование и удаление сотрудников доступно только в режиме админа. + Войти. +
+{% endif %} + +
+ + + {% if request.state.is_admin %}{% endif %} + + + {% for e in employees %} + {% if request.state.is_admin %} + + + + + + + + + {% else %} + + + + + + + {% endif %} + {% endfor %} + +
Имя@usernameChat IDПроектов
+ + + + +
+ + {% if e.tg_chat_id %} + ✓ + {% endif %} +
+ 		{{ e.projects|length }} + + +
+ +
+
{{ e.name }}{% if e.tg_username %}@{{ e.tg_username }}{% else %}—{% endif %} + {% if e.tg_chat_id %}{{ e.tg_chat_id }} ✓ + {% else %}не задан{% endif %} + {{ e.projects|length }}
+
+ +{% endblock %} diff --git a/app/templates/project_detail.html b/app/templates/project_detail.html new file mode 100644 index 0000000..1c6bf88 --- /dev/null +++ b/app/templates/project_detail.html @@ -0,0 +1,136 @@ +{% extends "base.html" %} +{% block title %}{{ project.title }} — DLD Monitor{% endblock %} +{% block content %} + +
+
+

{{ project.title }}

+
+ {% if project.deal_type.value == 'sale' %} + Продажа + {% else %} + Аренда + {% endif %} + · Владелец: {{ project.owner.name }} + {% if project.last_checked_at %} + · Проверено: {{ project.last_checked_at | msk }} МСК + {% endif %} +
+
+ {% if project.building %}🏢 {{ project.building }}{% endif %} + {% if project.bedrooms is not none %} · 🛏️ {{ project.bedrooms }} BR{% endif %} + {% if project.size_sqft %} · 📐 {{ "{:,.0f}".format(project.size_sqft).replace(",", " ") }} sqft{% endif %} + {% if project.dld_permit %} · permit: {{ project.dld_permit }}{% endif %} +
+ {% if project.our_price %} +
Наша цена: {{ "{:,.0f}".format(project.our_price).replace(",", " ") }} AED
+ {% endif %} + {% if project.our_url %} +
Наше объявление: {{ project.our_url }}
+ {% endif %} + {% if project.notes %}
{{ project.notes }}
{% endif %} +
+
+
+ +
+ {% if request.state.is_admin %} +
+ +
+ {% endif %} +
+
+ +{% if error %} +
{{ error }}
+{% endif %} +{% if message %} +
{{ message }}
+{% endif %} + +
+
+
+ +
+
+ +
+
+ {% if project.our_url %} +
+ + 🔍 Подобрать похожие на PropertyFinder + + + — по зданию из вашего объявления{% if project.bedrooms is not none %}, {{ project.bedrooms }} BR{% endif %}; + совпадения по DLD permit — первыми. Займёт ~15–20 сек. + +
+ {% else %} +
+ 🔍 «Подобрать похожие» появится, когда у проекта заполнены наше объявление (URL) и спальни. +
+ {% endif %} +
+ +
Отслеживаемые конкуренты ({{ project.listings|length }})
+ +{% if not project.listings %} +
+ Пока ничего не отслеживается. Добавьте URL объявления конкурента выше. +
+{% else %} + {% for l in project.listings %} +
+
+
+
+ + {{ 'PropertyFinder' if l.source.value == 'propertyfinder' else 'Bayut' }} + + {% if l.status.value == 'removed' %}Удалено{% endif %} +
+ {{ l.title or 'без названия' }} +
+
+ Брокер: {{ l.agent_name or '—' }} ({{ l.agency_name or '—' }})
+ Добавлено: {{ l.first_seen_at | msk }} · + Последний раз видели активным: {{ l.last_seen_at | msk }} (МСК) +
+
+
+
+ {% if l.current_price %} + {{ "{:,.0f}".format(l.current_price).replace(",", " ") }} {{ l.currency or 'AED' }} + {% else %}—{% endif %} +
+ {% if request.state.is_admin %} +
+ +
+ {% endif %} +
+
+ + {% if l.price_history|length > 1 %} +
+ История цены ({{ l.price_history|length }} записей) +
    + {% for h in l.price_history %} +
  • {{ h.recorded_at | msk }} — + {% if h.price %}{{ "{:,.0f}".format(h.price).replace(",", " ") }} AED{% else %}—{% endif %}
    • + {% endfor %} +
+
+ {% endif %} +
+
+ {% endfor %} +{% endif %} + +{% endblock %} diff --git a/app/templates/project_form.html b/app/templates/project_form.html new file mode 100644 index 0000000..2b87510 --- /dev/null +++ b/app/templates/project_form.html @@ -0,0 +1,81 @@ +{% extends "base.html" %} +{% block title %}Новый проект — DLD Monitor{% endblock %} +{% block content %} + +

Новый проект

+ +{% if no_employees %} +
+ Сначала добавьте хотя бы одного сотрудника — он будет получать уведомления в Telegram. +
+{% endif %} + +
+
+ + +
+ +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+

+ Поля ниже опциональны, но позволят системе предлагать «похожие» объявления конкурентов с PF и Bayut. +

+ +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+ + +
+
+ + +
+
+ +
+ + +
+ + + Отмена +
+ +{% endblock %} diff --git a/app/templates/projects_list.html b/app/templates/projects_list.html new file mode 100644 index 0000000..e77d502 --- /dev/null +++ b/app/templates/projects_list.html @@ -0,0 +1,67 @@ +{% extends "base.html" %} +{% block title %}Проекты — DLD Monitor{% endblock %} +{% block content %} + +
+

Наши проекты

+ + Новый проект +
+ +{% if projects %} +
+
{{ projects|length }}
проектов
+
{{ projects|map(attribute='listings')|map('length')|sum }}
конкурентов
+
+{% endif %} + +{% if not projects %} +
+ Пока ни одного проекта. Добавьте первый. +
+{% else %} +
+ + + + + + + + + + + + + + + {% for p in projects %} + + + + + + + + + + + {% endfor %} + +
НазваниеЗданиеТипНаша ценаКонкурентыВладелецПоследняя проверка (МСК)
{{ p.title }} + {% if p.building %}{{ p.building }}{% else %}—{% endif %} + {% if p.bedrooms is not none %} · {{ p.bedrooms }}BR{% endif %} + + {% if p.deal_type.value == 'sale' %} + Продажа + {% else %} + Аренда + {% endif %} + {% if p.our_price %}{{ "{:,.0f}".format(p.our_price).replace(",", " ") }} AED{% else %}—{% endif %}{{ p.listings|length }}{{ p.owner.name if p.owner else '—' }}{{ p.last_checked_at | msk }} +
+ +
+
+
+{% endif %} + +{% endblock %} diff --git a/app/templates/suggest.html b/app/templates/suggest.html new file mode 100644 index 0000000..6f73d4b --- /dev/null +++ b/app/templates/suggest.html @@ -0,0 +1,138 @@ +{% extends "base.html" %} +{% block title %}Похожие — {{ project.title }}{% endblock %} + +{% block head %} + +{% endblock %} + +{% block content %} + +
+ ← к проекту +
+ +

Подсказки похожих объявлений

+

+ Ищем объявления в том же здании{% if project.bedrooms is not none %}, {{ project.bedrooms }} спален{% endif %}, + тип сделки: {{ project.deal_type.value }}. + {% if our_permit %}Совпадение по DLD permit ({{ our_permit }}) — это тот же объект, такие показаны первыми. {% endif %} + Отметьте подходящие объявления галочкой (можно несколько) и нажмите «Отслеживать выбранные» — они попадут в трекинг. +

+ +
+{% for src_key, src_label in [('propertyfinder', 'PropertyFinder'), ('bayut', 'Bayut')] %} +
+ {{ src_label }} + {% if src_key == 'bayut' and not bayut_enabled %} + — ⏸ временно отключён + {% else %} + — найдено {{ suggestions[src_key]|length }} + {% endif %} +
+ {% if src_key == 'bayut' and not bayut_enabled %} +
Bayut перешёл на защищённый рендеринг — поиск и трекинг временно недоступны.
+ {% elif not suggestions[src_key] %} +
Ничего не нашли (или площадка заблокировала запрос).
+ {% else %} + {% for s in suggestions[src_key] %} + {% set same_permit = our_permit and s.permit_number and s.permit_number == our_permit %} + + {% endfor %} + {% endif %} +{% endfor %} + +
+
+
+ + +
+ +
+
+
+ +{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/app/web.py b/app/web.py new file mode 100644 index 0000000..4a7aa31 --- /dev/null +++ b/app/web.py @@ -0,0 +1,386 @@ +"""FastAPI web app — UI for managing projects, competitor listings, employees.""" + +from __future__ import annotations + +from datetime import timedelta +from pathlib import Path + +from fastapi import Depends, FastAPI, Form, HTTPException, Request +from fastapi.responses import HTMLResponse, RedirectResponse +from urllib.parse import quote +from fastapi.templating import Jinja2Templates +from sqlalchemy.orm import Session, joinedload + +from app.auth import ( + ADMIN_COOKIE, + COOKIE_MAX_AGE, + admin_configured, + admin_token, + pin_ok, + request_is_admin, +) +from app.config import BASE_DIR, settings +from app.db import get_db, init_db +from app.models import CompetitorListing, DealType, Employee, Project +from app.services.monitor import ( + BAYUT_ENABLED, + add_competitor_url, + add_competitor_urls, + resolve_our_permit, + run_check_for_project, + suggest_similar, +) + +app = FastAPI(title="DLD Monitor") +templates = Jinja2Templates(directory=str(Path(BASE_DIR) / "app" / "templates")) + +# Timestamps are stored as naive UTC (datetime.utcnow). Show them in Moscow time +# (UTC+3) in the UI via the `msk` Jinja filter. +_MSK_OFFSET = timedelta(hours=3) + + +def _to_msk(dt, fmt: str = "%Y-%m-%d %H:%M"): + if dt is None: + return "—" + return (dt + _MSK_OFFSET).strftime(fmt) + + +templates.env.filters["msk"] = _to_msk + + +def web_path(path: str = "/") -> str: + base = settings.public_base_path.rstrip("/") + if not path.startswith("/"): + path = "/" + path + return f"{base}{path}" if base else path + + +templates.env.globals["url_path"] = web_path + + +@app.on_event("startup") +def _startup() -> None: + init_db() + + +@app.middleware("http") +async def _admin_state(request: Request, call_next): + """Expose admin status to every route and template (request.state.is_admin).""" + request.state.is_admin = request_is_admin(request) + request.state.admin_configured = admin_configured() + return await call_next(request) + + +@app.get("/healthz") +def healthz() -> dict[str, str]: + return {"status": "ok"} + + +def admin_required(request: Request) -> None: + """Dependency: block the request unless the session is in admin mode.""" + if not request.state.is_admin: + raise HTTPException(403, "Действие доступно только администратору. Войдите в режим админа.") + + +def _safe_next(next_url: str) -> str: + """Only allow same-site relative redirects after login.""" + return next_url if next_url.startswith("/") and not next_url.startswith("//") else "/" + + +# --- Admin session -------------------------------------------------------- + +@app.get("/admin/login", response_class=HTMLResponse) +def admin_login_page(request: Request, next: str = "/", error: str | None = None): + return templates.TemplateResponse( + "admin_login.html", + {"request": request, "next": _safe_next(next), "error": error, "flash": None}, + ) + + +@app.post("/admin/login") +def admin_login(request: Request, pin: str = Form(...), next: str = Form("/")): + dest = _safe_next(next) + if not pin_ok(pin): + return templates.TemplateResponse( + "admin_login.html", + {"request": request, "next": dest, "error": "Неверный PIN", "flash": None}, + status_code=401, + ) + resp = RedirectResponse(web_path(dest), status_code=303) + resp.set_cookie( + ADMIN_COOKIE, admin_token(), max_age=COOKIE_MAX_AGE, + httponly=True, samesite="lax", + ) + return resp + + +@app.post("/admin/logout") +def admin_logout(next: str = Form("/")): + resp = RedirectResponse(web_path(_safe_next(next)), status_code=303) + resp.delete_cookie(ADMIN_COOKIE) + return resp + + +def _opt_float(s: str) -> float | None: + s = (s or "").strip() + if not s: + return None + try: + return float(s) + except ValueError: + return None + + +def _opt_int(s: str) -> int | None: + s = (s or "").strip() + if not s: + return None + try: + return int(s) + except ValueError: + return None + + +# --- Projects ------------------------------------------------------------- + +@app.get("/", response_class=HTMLResponse) +def projects_list(request: Request, db: Session = Depends(get_db)): + projects = ( + db.query(Project) + .options(joinedload(Project.owner), joinedload(Project.listings)) + .order_by(Project.created_at.desc()) + .all() + ) + return templates.TemplateResponse( + "projects_list.html", {"request": request, "projects": projects, "flash": None} + ) + + +@app.get("/projects/new", response_class=HTMLResponse) +def new_project_form(request: Request, db: Session = Depends(get_db)): + employees = db.query(Employee).order_by(Employee.name).all() + return templates.TemplateResponse( + "project_form.html", + { + "request": request, + "employees": employees, + "no_employees": not employees, + "flash": None, + }, + ) + + +@app.post("/projects/new") +def create_project( + title: str = Form(...), + deal_type: str = Form(...), + owner_id: int = Form(...), + our_price: str = Form(""), + building: str = Form(""), + bedrooms: str = Form(""), + size_sqft: str = Form(""), + our_url: str = Form(""), + dld_permit: str = Form(""), + notes: str = Form(""), + db: Session = Depends(get_db), +): + owner = db.get(Employee, owner_id) + if not owner: + raise HTTPException(404, "Employee not found") + project = Project( + title=title.strip(), + deal_type=DealType(deal_type), + our_price=_opt_float(our_price), + building=building.strip() or None, + bedrooms=_opt_int(bedrooms), + size_sqft=_opt_float(size_sqft), + our_url=our_url.strip() or None, + dld_permit=dld_permit.strip() or None, + notes=notes.strip() or None, + owner_id=owner.id, + ) + db.add(project) + db.commit() + return RedirectResponse(web_path(f"/projects/{project.id}"), status_code=303) + + +@app.get("/projects/{project_id}", response_class=HTMLResponse) +def project_detail( + project_id: int, + request: Request, + error: str | None = None, + message: str | None = None, + db: Session = Depends(get_db), +): + project = ( + db.query(Project) + .options(joinedload(Project.owner), joinedload(Project.listings)) + .filter(Project.id == project_id) + .first() + ) + if not project: + raise HTTPException(404, "Project not found") + return templates.TemplateResponse( + "project_detail.html", + {"request": request, "project": project, "error": error, "message": message, "flash": None}, + ) + + +@app.post("/projects/{project_id}/check") +def project_check_now(project_id: int, db: Session = Depends(get_db)): + if not db.get(Project, project_id): + raise HTTPException(404, "Project not found") + db.close() + run_check_for_project(project_id) + return RedirectResponse(web_path(f"/projects/{project_id}"), status_code=303) + + +@app.post("/projects/{project_id}/delete") +def project_delete(project_id: int, db: Session = Depends(get_db), _: None = Depends(admin_required)): + project = db.get(Project, project_id) + if not project: + raise HTTPException(404, "Project not found") + db.delete(project) + db.commit() + return RedirectResponse(web_path("/"), status_code=303) + + +# --- Competitor listings (per project) ------------------------------------ + +@app.post("/projects/{project_id}/listings") +def add_listing(project_id: int, url: str = Form(...), db: Session = Depends(get_db)): + project = db.get(Project, project_id) + if not project: + raise HTTPException(404, "Project not found") + listing, err = add_competitor_url(db, project, url) + if err: + return RedirectResponse(web_path(f"/projects/{project_id}?error={quote(err)}"), status_code=303) + return RedirectResponse( + web_path(f"/projects/{project_id}?message=Добавлено"), status_code=303, + ) + + +@app.post("/projects/{project_id}/listings/bulk") +def add_listings_bulk( + project_id: int, + urls: list[str] = Form(default=[]), + db: Session = Depends(get_db), +): + project = db.get(Project, project_id) + if not project: + raise HTTPException(404, "Project not found") + if not urls: + return RedirectResponse( + web_path(f"/projects/{project_id}?error={quote('Ничего не выбрано')}"), + status_code=303, + ) + result = add_competitor_urls(db, project, urls) + parts = [f"Добавлено: {result['added']}"] + if result["skipped"]: + parts.append(f"уже были: {result['skipped']}") + if result["errors"]: + parts.append(f"ошибок: {len(result['errors'])}") + msg = " · ".join(parts) + return RedirectResponse( + web_path(f"/projects/{project_id}?message={quote(msg)}"), status_code=303, + ) + + +@app.post("/listings/{listing_id}/delete") +def listing_delete(listing_id: int, db: Session = Depends(get_db), _: None = Depends(admin_required)): + listing = db.get(CompetitorListing, listing_id) + if not listing: + raise HTTPException(404, "Listing not found") + project_id = listing.project_id + db.delete(listing) + db.commit() + return RedirectResponse(web_path(f"/projects/{project_id}"), status_code=303) + + +@app.get("/projects/{project_id}/suggest", response_class=HTMLResponse) +def project_suggest(project_id: int, request: Request, db: Session = Depends(get_db)): + project = ( + db.query(Project) + .options(joinedload(Project.listings)) + .filter(Project.id == project_id) + .first() + ) + if not project: + raise HTTPException(404, "Project not found") + our_permit = resolve_our_permit(project) + suggestions = suggest_similar(project, our_permit=our_permit) + return templates.TemplateResponse( + "suggest.html", + { + "request": request, + "project": project, + "suggestions": suggestions, + "our_permit": our_permit, + "bayut_enabled": BAYUT_ENABLED, + "flash": None, + }, + ) + + +# --- Employees ------------------------------------------------------------ + +@app.get("/employees", response_class=HTMLResponse) +def employees_page(request: Request, db: Session = Depends(get_db)): + employees = ( + db.query(Employee).options(joinedload(Employee.projects)).order_by(Employee.name).all() + ) + return templates.TemplateResponse( + "employees.html", {"request": request, "employees": employees, "flash": None} + ) + + +@app.post("/employees") +def employee_create( + name: str = Form(...), + tg_username: str = Form(""), + db: Session = Depends(get_db), +): + e = Employee(name=name.strip(), tg_username=tg_username.strip().lstrip("@") or None) + db.add(e) + db.commit() + return RedirectResponse(web_path("/employees"), status_code=303) + + +@app.post("/employees/{employee_id}/update") +def employee_update( + employee_id: int, + name: str = Form(...), + tg_username: str = Form(""), + tg_chat_id: str = Form(""), + db: Session = Depends(get_db), + _: None = Depends(admin_required), +): + e = db.get(Employee, employee_id) + if not e: + raise HTTPException(404, "Employee not found") + e.name = name.strip() + e.tg_username = tg_username.strip().lstrip("@") or None + new_chat_id = tg_chat_id.strip() or None + if new_chat_id and new_chat_id != e.tg_chat_id: + clash = ( + db.query(Employee) + .filter(Employee.tg_chat_id == new_chat_id, Employee.id != e.id) + .first() + ) + if clash: + raise HTTPException(400, f"chat_id {new_chat_id} уже привязан к сотруднику «{clash.name}»") + e.tg_chat_id = new_chat_id + db.commit() + return RedirectResponse(web_path("/employees"), status_code=303) + + +@app.post("/employees/{employee_id}/delete") +def employee_delete(employee_id: int, db: Session = Depends(get_db), _: None = Depends(admin_required)): + e = db.get(Employee, employee_id) + if not e: + raise HTTPException(404, "Employee not found") + if e.projects: + raise HTTPException(400, "Сотрудник связан с проектами — сначала переназначьте или удалите проекты") + db.delete(e) + db.commit() + return RedirectResponse(web_path("/employees"), status_code=303) diff --git a/diagnose.py b/diagnose.py new file mode 100644 index 0000000..30f3362 --- /dev/null +++ b/diagnose.py @@ -0,0 +1,89 @@ +r"""Одноразовая диагностика: проверяет всю цепочку мониторинга. +Запуск: & "<...>\.venv\Scripts\python.exe" diagnose.py +Только чтение БД + проверка токена TG + живой re-fetch. Сообщения НЕ шлёт.""" +from __future__ import annotations + +import sys + +import httpx + +from app.config import settings +from app.db import SessionLocal +from app.models import CompetitorListing, Employee, PriceHistory, Project +from app.services.monitor import check_project + + +def line(c: str = "-") -> None: + print(c * 60) + + +def main() -> None: + db = SessionLocal() + try: + # --- 1. Содержимое БД --- + line("=") + print("1) ДАННЫЕ В БАЗЕ") + line("=") + projects = db.query(Project).all() + if not projects: + print(" ! В базе нет ни одного проекта.") + return + for p in projects: + owner = p.owner + chat = owner.tg_chat_id if owner else None + chat_str = chat if chat else "НЕ ЗАДАН (уведомления не дойдут!)" + print(f"\n Проект #{p.id}: {p.title}") + print(f" тип: {p.deal_type.value} | наша цена: {p.our_price}") + print(f" владелец: {owner.name if owner else '—'} | tg_chat_id: {chat_str}") + print(f" последняя проверка: {p.last_checked_at or 'ещё не было'}") + listings = p.listings + print(f" отслеживаемых объявлений конкурентов: {len(listings)}") + for l in listings: + hist = db.query(PriceHistory).filter(PriceHistory.listing_id == l.id).count() + price = f"{l.current_price:,.0f} {l.currency}".replace(",", " ") if l.current_price else "БЕЗ ЦЕНЫ" + print(f" - [{l.source.value}] {l.status.value} | {price} | история: {hist} зап.") + print(f" {(l.title or 'без названия')[:70]}") + print(f" брокер: {l.agency_name or '—'} | {l.url}") + + # --- 2. Telegram токен --- + line("=") + print("2) TELEGRAM") + line("=") + if not settings.tg_bot_token: + print(" ! TG_BOT_TOKEN не задан в .env") + else: + try: + r = httpx.get( + f"https://api.telegram.org/bot{settings.tg_bot_token}/getMe", + timeout=15.0, + ) + if r.status_code == 200 and r.json().get("ok"): + bot = r.json()["result"] + print(f" OK токен валиден: @{bot.get('username')} ({bot.get('first_name')})") + else: + print(f" ! getMe вернул {r.status_code}: {r.text}") + except Exception as e: + print(f" ! Ошибка связи с Telegram: {e}") + print(f" ADMIN_CHAT_ID: {settings.admin_chat_id or 'не задан'}") + + # --- 3. Живой re-fetch (то, что делает сканер раз в 4 ч) --- + line("=") + print("3) ЖИВАЯ ПЕРЕПРОВЕРКА (re-fetch всех объявлений)") + line("=") + for p in projects: + print(f"\n Проект #{p.id}: {p.title} — проверяю {len(p.listings)} объявл...") + changes = check_project(db, p) + if changes: + print(f" обнаружено изменений: {len(changes)} (в реальном прогоне ушли бы в TG):") + for c in changes: + print(" " + c.replace("\n", " | ").replace("", "").replace("", "")) + else: + print(" изменений нет — все объявления успешно перезагружены, цены прежние.") + line("=") + print("Готово. Если у объявлений есть цены и статус active, а токен валиден — цепочка рабочая.") + finally: + db.close() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6683ffe --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,37 @@ +services: + web: + build: . + image: dld-permit-monitor:latest + container_name: dld-web + restart: unless-stopped + env_file: .env + environment: + WEB_HOST: 0.0.0.0 + WEB_PORT: 8000 + ports: + - "8080:8000" + volumes: + - ./data:/app/data + command: ["python", "run_web.py"] + + bot: + image: dld-permit-monitor:latest + depends_on: + - web + container_name: dld-bot + restart: unless-stopped + env_file: .env + volumes: + - ./data:/app/data + command: ["python", "-m", "app.bot"] + + scheduler: + image: dld-permit-monitor:latest + depends_on: + - web + container_name: dld-scheduler + restart: unless-stopped + env_file: .env + volumes: + - ./data:/app/data + command: ["python", "-m", "app.scheduler"] diff --git a/k8s/configmap.yaml b/k8s/configmap.yaml new file mode 100644 index 0000000..a944996 --- /dev/null +++ b/k8s/configmap.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: monitoring-pf-config + namespace: monitoring-pf +data: + WEB_HOST: "0.0.0.0" + WEB_PORT: "8000" + PUBLIC_BASE_PATH: "/api/monitoring-pf" + DATABASE_URL: "sqlite:////data/monitor.db" + SCRAPE_INTERVAL_HOURS: "4" diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml new file mode 100644 index 0000000..6a4d14d --- /dev/null +++ b/k8s/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: monitoring-pf + +resources: + - namespace.yaml + - configmap.yaml + - secrets.yaml + - server-deployment.yaml + - server-service.yaml diff --git a/k8s/namespace.yaml b/k8s/namespace.yaml new file mode 100644 index 0000000..7839b37 --- /dev/null +++ b/k8s/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring-pf diff --git a/k8s/secrets.yaml b/k8s/secrets.yaml new file mode 100644 index 0000000..d89ae36 --- /dev/null +++ b/k8s/secrets.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: monitoring-pf-secrets + namespace: monitoring-pf +type: Opaque +stringData: + TG_BOT_TOKEN: "CHANGE_ME" + ADMIN_CHAT_ID: "" + ADMIN_PIN: "CHANGE_ME" diff --git a/k8s/server-deployment.yaml b/k8s/server-deployment.yaml new file mode 100644 index 0000000..d9ffe50 --- /dev/null +++ b/k8s/server-deployment.yaml @@ -0,0 +1,107 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: monitoring-pf-data + namespace: monitoring-pf +spec: + accessModes: ["ReadWriteOnce"] + storageClassName: local-path + resources: + requests: + storage: 5Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: monitoring-pf-server + namespace: monitoring-pf +spec: + replicas: 1 + selector: + matchLabels: + app: monitoring-pf-server + template: + metadata: + labels: + app: monitoring-pf-server + spec: + terminationGracePeriodSeconds: 20 + securityContext: + fsGroup: 1000 + containers: + - name: web + image: localhost:30300/admin/monitoring-pf-server:latest + command: ["python", "run_web.py"] + ports: + - containerPort: 8000 + envFrom: + - configMapRef: + name: monitoring-pf-config + - secretRef: + name: monitoring-pf-secrets + volumeMounts: + - name: app-data + mountPath: /data + startupProbe: + httpGet: + path: /healthz + port: 8000 + periodSeconds: 5 + failureThreshold: 30 + livenessProbe: + httpGet: + path: /healthz + port: 8000 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /healthz + port: 8000 + periodSeconds: 5 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 800m + memory: 768Mi + - name: bot + image: localhost:30300/admin/monitoring-pf-server:latest + command: ["python", "-m", "app.bot"] + envFrom: + - configMapRef: + name: monitoring-pf-config + - secretRef: + name: monitoring-pf-secrets + volumeMounts: + - name: app-data + mountPath: /data + resources: + requests: + cpu: 25m + memory: 96Mi + limits: + cpu: 300m + memory: 384Mi + - name: scheduler + image: localhost:30300/admin/monitoring-pf-server:latest + command: ["python", "-m", "app.scheduler"] + envFrom: + - configMapRef: + name: monitoring-pf-config + - secretRef: + name: monitoring-pf-secrets + volumeMounts: + - name: app-data + mountPath: /data + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 600m + memory: 768Mi + volumes: + - name: app-data + persistentVolumeClaim: + claimName: monitoring-pf-data diff --git a/k8s/server-service.yaml b/k8s/server-service.yaml new file mode 100644 index 0000000..fda79e6 --- /dev/null +++ b/k8s/server-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: monitoring-pf-server + namespace: monitoring-pf + annotations: + portal.estateliga.work/enabled: "true" + portal.estateliga.work/name: "Мониторинг PF" + portal.estateliga.work/description: "Мониторинг объявлений PropertyFinder/Bayut по DLD Permit" + portal.estateliga.work/icon: "pulse" + portal.estateliga.work/path: "/api/monitoring-pf" + portal.estateliga.work/code: "monitoring_pf" +spec: + selector: + app: monitoring-pf-server + ports: + - port: 80 + targetPort: 8000 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a878a49 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +fastapi==0.115.6 +uvicorn[standard]==0.32.1 +jinja2==3.1.4 +python-multipart==0.0.20 +sqlalchemy==2.0.36 +httpx==0.28.1 +brotli==1.2.0 # lets httpx decode Content-Encoding: br (Bayut serves Brotli) +beautifulsoup4==4.12.3 +lxml==5.3.0 +apscheduler==3.11.0 +python-telegram-bot==21.9 +python-dotenv==1.0.1 +pydantic==2.10.4 +pydantic-settings==2.7.0 diff --git a/run_bot.bat b/run_bot.bat new file mode 100644 index 0000000..4ebe5f9 --- /dev/null +++ b/run_bot.bat @@ -0,0 +1,4 @@ +@echo off +cd /d "%~dp0" +".venv\Scripts\python.exe" -m app.bot +pause diff --git a/run_scheduler.bat b/run_scheduler.bat new file mode 100644 index 0000000..4be836a --- /dev/null +++ b/run_scheduler.bat @@ -0,0 +1,4 @@ +@echo off +cd /d "%~dp0" +".venv\Scripts\python.exe" -m app.scheduler +pause diff --git a/run_web.bat b/run_web.bat new file mode 100644 index 0000000..c3a27e9 --- /dev/null +++ b/run_web.bat @@ -0,0 +1,4 @@ +@echo off +cd /d "%~dp0" +".venv\Scripts\python.exe" run_web.py +pause diff --git a/run_web.py b/run_web.py new file mode 100644 index 0000000..716d105 --- /dev/null +++ b/run_web.py @@ -0,0 +1,13 @@ +"""Convenience launcher for the web UI: `python run_web.py`.""" + +import uvicorn + +from app.config import settings + +if __name__ == "__main__": + uvicorn.run( + "app.web:app", + host=settings.web_host, + port=settings.web_port, + reload=False, + )