Add monitoring PF service

.dockerignore

@@ -0,0 +1,23 @@
+.venv/
+__pycache__/
+**/__pycache__/
+*.pyc
+*.pyo
+.git/
+.gitignore
+.idea/
+.vscode/
+.claude/
+.DS_Store
+data/
+*.db
+*.db-journal
+*.log
+*.bat
+diagnose.py
+README.md
+SESSION_NOTES.md
+Dockerfile
+docker-compose.yml
+.dockerignore
+.env

.gitea/workflows/deploy.yaml

@@ -0,0 +1,59 @@
+name: Build and Deploy
+
+on:
+  push:
+    branches: [main]
+
+env:
+  INTERNAL_REGISTRY: gitea-http.gitea.svc.cluster.local:3000
+  NODE_REGISTRY: localhost:30300
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Docker CLI
+        run: |
+          curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-27.5.1.tgz \
+            | tar xz --strip-components=1 -C /usr/local/bin docker/docker
+          docker version
+
+      - name: Install kubectl
+        run: |
+          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+          chmod +x kubectl
+          mv kubectl /usr/local/bin/
+          kubectl version --client
+
+      - name: Login to Gitea Registry
+        run: |
+          echo "${{ secrets.REGISTRY_PASSWORD }}" | \
+            docker login ${{ env.INTERNAL_REGISTRY }} \
+              -u ${{ secrets.REGISTRY_USERNAME }} --password-stdin
+
+      - name: Build and push server
+        run: |
+          docker build -f Dockerfile \
+            -t ${{ env.INTERNAL_REGISTRY }}/admin/monitoring-pf-server:${{ github.sha }} \
+            -t ${{ env.INTERNAL_REGISTRY }}/admin/monitoring-pf-server:latest \
+            .
+          docker push ${{ env.INTERNAL_REGISTRY }}/admin/monitoring-pf-server:${{ github.sha }}
+          docker push ${{ env.INTERNAL_REGISTRY }}/admin/monitoring-pf-server:latest
+
+      - name: Deploy to Kubernetes
+        env:
+          KUBECONFIG: /kubeconfig/config
+        run: |
+          kubectl apply -f k8s/namespace.yaml
+          kubectl apply -f k8s/secrets.yaml
+          kubectl apply -f k8s/configmap.yaml
+          kubectl apply -f k8s/server-deployment.yaml
+          kubectl apply -f k8s/server-service.yaml
+          kubectl -n monitoring-pf set image deployment/monitoring-pf-server \
+            web=${{ env.NODE_REGISTRY }}/admin/monitoring-pf-server:${{ github.sha }} \
+            bot=${{ env.NODE_REGISTRY }}/admin/monitoring-pf-server:${{ github.sha }} \
+            scheduler=${{ env.NODE_REGISTRY }}/admin/monitoring-pf-server:${{ github.sha }}
+          kubectl -n monitoring-pf rollout status deployment/monitoring-pf-server --timeout=180s

.gitignore

@@ -0,0 +1,10 @@
+.venv/
+__pycache__/
+*.pyc
+.env
+data/
+*.db
+*.db-journal
+.idea/
+.vscode/
+*.log

Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.12-slim
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+
+WORKDIR /app
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends tini \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt ./
+RUN pip install -r requirements.txt
+
+COPY app ./app
+COPY run_web.py ./
+
+RUN mkdir -p /app/data
+
+EXPOSE 8000
+
+ENTRYPOINT ["/usr/bin/tini", "--"]
+CMD ["python", "run_web.py"]

README.md

@@ -0,0 +1,143 @@
+# DLD Monitor
+
+Внутренний инструмент для агентства недвижимости в Дубае: мониторит цены объявлений конкурентов на **PropertyFinder.ae** и **Bayut.com** по DLD Permit Number, шлёт уведомления в Telegram при:
+
+- 📈📉 изменении цены конкурента,
+- ❌ удалении объявления (404 / withdrawn),
+- 🆕 появлении нового объявления с тем же permit (новый брокер выставил ту же квартиру).
+
+## Архитектура
+
+```
+┌──────────────┐    ┌──────────────────┐    ┌─────────────────┐
+│  Web UI      │    │  Scheduler       │    │  Telegram Bot   │
+│  (FastAPI)   │    │  (APScheduler)   │    │  (polling)      │
+│  add project │    │  every N hours   │    │  /start /check  │
+└──────┬───────┘    └────────┬─────────┘    └────────┬────────┘
+       │                     │                       │
+       └──────────────┬──────┴───────────────────────┘
+                     ▼
+              ┌────────────────┐
+              │   monitor      │
+              │   service      │  ← скрапит PF и Bayut, пишет в SQLite
+              └────────────────┘
+                     │
+                     ▼  уведомления в TG конкретному employee
+```
+
+## Локальный запуск (Windows)
+
+### 1. Создать виртуальное окружение и поставить зависимости
+
+```powershell
+python -m venv .venv
+.\.venv\Scripts\Activate.ps1
+pip install -r requirements.txt
+```
+
+### 2. Получить токен Telegram-бота
+
+Уже есть? Отлично. Если нет:
+1. Откройте Telegram, найдите [@BotFather](https://t.me/BotFather).
+2. Отправьте `/newbot`, придумайте имя.
+3. Скопируйте токен вида `123456:ABC-DEF…`.
+
+### 3. Создать `.env`
+
+```powershell
+Copy-Item .env.example .env
+notepad .env
+```
+
+В `.env` вставьте:
+
+```
+TG_BOT_TOKEN=ваш_токен_от_botfather
+SCRAPE_INTERVAL_HOURS=4
+ADMIN_CHAT_ID=  # опционально — куда слать системные ошибки
+```
+
+### 4. Запустить три процесса (в **трёх** разных окнах PowerShell)
+
+**Окно 1 — веб-интерфейс:**
+```powershell
+.\.venv\Scripts\Activate.ps1
+python run_web.py
+```
+Откройте http://127.0.0.1:8000
+
+**Окно 2 — Telegram-бот:**
+```powershell
+.\.venv\Scripts\Activate.ps1
+python -m app.bot
+```
+
+**Окно 3 — фоновый сканер:**
+```powershell
+.\.venv\Scripts\Activate.ps1
+python -m app.scheduler
+```
+
+## Первое использование
+
+1. Откройте бота в Telegram и отправьте `/start` — он зарегистрирует ваш `chat_id`.
+2. В веб-UI перейдите в **Сотрудники** — убедитесь, что вы там есть с ✓ TG.
+3. Нажмите **+ Новый проект**, заполните:
+   - Название (например: «Marina Pinnacle 1502, 2BR»)
+   - DLD Permit Number (Trakheesi)
+   - Тип сделки (продажа/аренда)
+   - Владелец = вы
+4. На странице проекта нажмите **Проверить сейчас** — система найдёт все объявления конкурентов с этим permit на PF и Bayut.
+5. Дальше фоновый сканер сам будет проверять каждые `SCRAPE_INTERVAL_HOURS` часов и слать уведомления в Telegram.
+
+## Команды бота
+
+- `/start` — подключить себя как сотрудника (запоминает chat_id)
+- `/list` — список ваших проектов
+- `/check` — запустить проверку всех ваших проектов сейчас
+- `/whoami` — показать свой chat_id
+
+## Структура
+
+```
+app/
+├── config.py            настройки из .env
+├── db.py                SQLAlchemy engine + session
+├── models.py            Employee, Project, CompetitorListing, PriceHistory
+├── web.py               FastAPI роуты и UI
+├── bot.py               Telegram-бот
+├── scheduler.py         APScheduler фоновый сканер
+├── scrapers/
+│   ├── base.py          httpx + парсинг __NEXT_DATA__
+│   ├── propertyfinder.py
+│   └── bayut.py
+├── services/
+│   ├── monitor.py       детект изменений, основная бизнес-логика
+│   └── notifier.py      отправка в TG
+└── templates/           Jinja2 (Bootstrap 5)
+data/
+└── monitor.db           SQLite (создаётся автоматически)
+```
+
+## Перенос на сервер
+
+Когда придёт время — нужно:
+1. Поставить Python 3.11+ на сервер (Linux).
+2. Скопировать репозиторий, `pip install -r requirements.txt`.
+3. Создать `.env`.
+4. Поставить три процесса под `systemd`:
+   - `dld-monitor-web.service` → `python run_web.py`
+   - `dld-monitor-bot.service` → `python -m app.bot`
+   - `dld-monitor-scheduler.service` → `python -m app.scheduler`
+5. Поставить nginx + TLS перед веб-портом (8000).
+
+## Возможные проблемы скрапинга
+
+PF/Bayut могут начать блокировать запросы при частых обращениях. Признаки:
+- В логах сканера видны `Blocked by site (403/429)`.
+- Поиск возвращает 0 объявлений, хотя они должны быть.
+
+Что делать:
+1. Уменьшите `SCRAPE_INTERVAL_HOURS` (реже = меньше риска).
+2. Если не помогает — добавьте Playwright (headless-браузер). Заготовка: `pip install playwright && playwright install chromium`, затем заменить `fetch_html` на запуск через Playwright.
+3. Опционально — прокси.

SESSION_NOTES.md

@@ -0,0 +1,90 @@
+# Журнал сессии (2026-05-25)
+
+## Что построили
+
+Внутренний мониторинг цен конкурентов на propertyfinder.ae и bayut.com для HOME LIGA REAL ESTATE.
+
+**Стек:** FastAPI + Jinja2 (Bootstrap 5) + SQLAlchemy/SQLite + APScheduler + python-telegram-bot + httpx/BS4.
+
+**Три процесса** (каждый — свой .bat-лаунчер):
+- `run_web.bat` — веб-UI на http://127.0.0.1:8000
+- `run_bot.bat` — Telegram-бот (polling)
+- `run_scheduler.bat` — фоновый сканер каждые `SCRAPE_INTERVAL_HOURS` часов
+
+## Эволюция архитектуры
+
+### Первая попытка (отвергнута)
+**Идея:** сотрудник вводит DLD Permit Number своего объекта → система автоматически ищет «то же permit» на PF и Bayut → находит объявления конкурентов.
+
+**Почему не сработало:**
+1. В Дубае **каждый брокер получает свой permit на свою публикацию**. Два брокера, рекламирующих одну квартиру = два разных permit. Permit не идентифицирует физический объект — он идентифицирует конкретное объявление конкретного брокера.
+2. PF на странице объявления показывает permit **картинкой** через сервис верификации, не plain text (anti-scraping).
+3. PF search `?q=<permit>` — free-text по названию/описанию, не структурированный фильтр.
+
+### Финальная архитектура
+**Manual URL list + опциональные подсказки:**
+1. Сотрудник создаёт проект (название, тип сделки, владелец, опц. building/bedrooms/sqft).
+2. На странице проекта **вручную вставляет URL** объявления конкурента → система делает single-page fetch, парсит `__NEXT_DATA__`, добавляет в трекинг.
+3. Если указано здание — кнопка «🔍 Подобрать похожие» ищет на PF/Bayut по `building + bedrooms` и предлагает кандидатов с кнопкой «+ Отслеживать».
+4. Каждые 4 часа фоновый сканер делает refetch каждого отслеживаемого URL → детектит:
+   - 📈📉 изменение цены
+   - ❌ удаление (URL отдаёт 404)
+   - ♻️ возвращение из удалённого статуса
+5. Уведомления — в Telegram личкой владельцу проекта.
+
+## Модель данных
+
+- `Employee` — name, tg_chat_id (опц.), tg_username
+- `Project` — title, deal_type, owner_id, our_price, building, bedrooms, size_sqft, our_url, dld_permit (все после `owner` — опционально)
+- `CompetitorListing` — source (PF|Bayut), external_id, url, current_price, status (active|removed), agent_name, agency_name, first_seen, last_seen
+- `PriceHistory` — listing_id, price, recorded_at
+
+## Как подключиться сотруднику
+
+1. В TG найти бота → отправить `/start`.
+2. Отправить `/whoami` → бот пришлёт chat_id.
+3. В вебе http://127.0.0.1:8000/employees → найти/создать запись → вставить chat_id → Сохранить.
+
+## Известные ограничения
+
+- **PF/Bayut могут блокировать** при частых запросах (видно как `Blocked by site (403/429)` в логах). Решение — увеличить интервал; если уже не помогает — добавить Playwright fallback.
+- **Подсказки эвристические**: ищем по совпадению building name в title + bedrooms-фильтр. Могут попасть «другие квартиры в том же здании» — поэтому добавление в трекинг через ручное подтверждение.
+- **Permit как plain text** на PF не отдаётся. Если когда-нибудь понадобится — нужен OCR на verification-image.
+
+## Что в .env
+
+```
+TG_BOT_TOKEN=<токен от @BotFather>
+SCRAPE_INTERVAL_HOURS=4
+ADMIN_CHAT_ID=  # опц.
+```
+
+## Структура проекта
+
+```
+DLD Permit Number/
+├── run_web.bat, run_bot.bat, run_scheduler.bat
+├── run_web.py
+├── requirements.txt
+├── .env (не в git — содержит токен)
+├── data/monitor.db (создаётся автоматически)
+├── app/
+│   ├── config.py     ← settings + резолвит относительные SQLite-пути в абсолютные
+│   ├── db.py, models.py
+│   ├── web.py        ← FastAPI: CRUD проектов, /listings add/delete, /suggest
+│   ├── bot.py        ← /start, /whoami, /list, /check
+│   ├── scheduler.py
+│   ├── scrapers/{base,propertyfinder,bayut}.py
+│   ├── services/{monitor,notifier}.py
+│   └── templates/    ← projects_list, project_form, project_detail, suggest, employees, base
+```
+
+## Что протестировать в первую очередь
+
+1. Удалить старый `data/monitor.db` (схема изменилась).
+2. Запустить три .bat файла.
+3. `/start` боту → `/whoami` → chat_id → вписать в Сотрудники.
+4. Создать тестовый проект (Aykon City Tower B, 2BR).
+5. Вставить URL реального объявления конкурента → проверить, что добавилось с ценой/брокером.
+6. Жмякнуть «Подобрать похожие» → посмотреть кандидатов.
+7. Жмякнуть «Проверить сейчас» → если цена не менялась, изменений не будет (это нормально); для теста алертов можно поменять цену в БД руками или подождать реального изменения.

app/auth.py

@@ -0,0 +1,45 @@
+"""Lightweight admin gate for the web UI.
+
+A single shared PIN (`ADMIN_PIN` in .env) unlocks destructive actions
+(deleting projects/competitors/employees, editing employees) for the browser
+session via an HMAC-signed cookie. No per-user accounts — this is an internal
+localhost tool, not a public app.
+
+If `ADMIN_PIN` is empty the gate is OPEN (nothing restricted) so the tool is
+never bricked by a missing setting; set a PIN to actually restrict.
+"""
+from __future__ import annotations
+
+import hashlib
+import hmac
+
+from starlette.requests import Request
+
+from app.config import settings
+
+ADMIN_COOKIE = "dld_admin"
+COOKIE_MAX_AGE = 60 * 60 * 8  # 8 hours
+
+
+def admin_configured() -> bool:
+    return bool(settings.admin_pin)
+
+
+def admin_token() -> str | None:
+    """The cookie value that proves admin: HMAC(pin, marker). None if no PIN."""
+    if not settings.admin_pin:
+        return None
+    return hmac.new(settings.admin_pin.encode(), b"dld-admin-v1", hashlib.sha256).hexdigest()
+
+
+def pin_ok(pin: str) -> bool:
+    """Constant-time check of a submitted PIN against the configured one."""
+    return bool(settings.admin_pin) and hmac.compare_digest((pin or "").strip(), settings.admin_pin)
+
+
+def request_is_admin(request: Request) -> bool:
+    if not settings.admin_pin:
+        return True  # gate not configured → open
+    token = request.cookies.get(ADMIN_COOKIE)
+    expected = admin_token()
+    return bool(token and expected and hmac.compare_digest(token, expected))

app/bot.py

@@ -0,0 +1,182 @@
+"""Telegram bot — registers employees by chat_id and lets them trigger checks.
+
+Run as a separate process: `python -m app.bot`.
+
+Bot commands (set via @BotFather → /setcommands):
+    start  - Подключить себя как сотрудника
+    list   - Список своих проектов
+    check  - Проверить все мои проекты сейчас
+    whoami - Показать свой chat_id
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+
+from sqlalchemy.orm import joinedload
+from telegram import Update
+from telegram.ext import (
+    Application,
+    CommandHandler,
+    ContextTypes,
+)
+
+from app.config import settings
+from app.db import SessionLocal, init_db
+from app.models import Employee, Project
+from app.services.monitor import run_check_for_project
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+logger = logging.getLogger(__name__)
+
+
+async def cmd_start(update: Update, _: ContextTypes.DEFAULT_TYPE) -> None:
+    if not update.effective_user or not update.effective_chat:
+        return
+    user = update.effective_user
+    chat_id = str(update.effective_chat.id)
+    username = user.username
+
+    db = SessionLocal()
+    try:
+        existing = (
+            db.query(Employee).filter(Employee.tg_chat_id == chat_id).first()
+        )
+        if existing:
+            await update.message.reply_text(
+                f"✅ Вы уже подключены как <b>{existing.name}</b>.\n"
+                f"chat_id: <code>{chat_id}</code>",
+                parse_mode="HTML",
+            )
+            return
+
+        # Try to find by username (admin pre-created employee w/o chat_id)
+        if username:
+            placeholder = (
+                db.query(Employee)
+                .filter(Employee.tg_username == username, Employee.tg_chat_id.is_(None))
+                .first()
+            )
+            if placeholder:
+                placeholder.tg_chat_id = chat_id
+                db.commit()
+                await update.message.reply_text(
+                    f"✅ Привет, <b>{placeholder.name}</b>! Вы успешно подключены.\n"
+                    f"Уведомления будут приходить сюда.",
+                    parse_mode="HTML",
+                )
+                return
+
+        # Create a new employee record from this user
+        name = (user.full_name or username or f"user_{chat_id}").strip()
+        e = Employee(name=name, tg_chat_id=chat_id, tg_username=username)
+        db.add(e)
+        db.commit()
+        await update.message.reply_text(
+            f"👋 Привет, <b>{name}</b>! Вы зарегистрированы как сотрудник.\n"
+            f"Откройте веб-интерфейс и создайте проекты, чтобы получать уведомления.\n"
+            f"chat_id: <code>{chat_id}</code>",
+            parse_mode="HTML",
+        )
+    finally:
+        db.close()
+
+
+async def cmd_whoami(update: Update, _: ContextTypes.DEFAULT_TYPE) -> None:
+    if not update.effective_chat:
+        return
+    chat_id = str(update.effective_chat.id)
+    db = SessionLocal()
+    try:
+        e = db.query(Employee).filter(Employee.tg_chat_id == chat_id).first()
+        if e:
+            await update.message.reply_text(
+                f"Вы: <b>{e.name}</b>\nchat_id: <code>{chat_id}</code>",
+                parse_mode="HTML",
+            )
+        else:
+            await update.message.reply_text(
+                f"Вы пока не подключены. Отправьте /start.\nchat_id: <code>{chat_id}</code>",
+                parse_mode="HTML",
+            )
+    finally:
+        db.close()
+
+
+async def cmd_list(update: Update, _: ContextTypes.DEFAULT_TYPE) -> None:
+    if not update.effective_chat:
+        return
+    chat_id = str(update.effective_chat.id)
+    db = SessionLocal()
+    try:
+        e = (
+            db.query(Employee)
+            .options(joinedload(Employee.projects))
+            .filter(Employee.tg_chat_id == chat_id)
+            .first()
+        )
+        if not e:
+            await update.message.reply_text("Сначала /start.")
+            return
+        if not e.projects:
+            await update.message.reply_text("У вас пока нет проектов.")
+            return
+        lines = [f"<b>Ваши проекты ({len(e.projects)}):</b>"]
+        for p in e.projects:
+            lines.append(
+                f"• #{p.id} {p.title} — <code>{p.dld_permit}</code> "
+                f"({p.deal_type.value})"
+            )
+        await update.message.reply_text("\n".join(lines), parse_mode="HTML")
+    finally:
+        db.close()
+
+
+async def cmd_check(update: Update, _: ContextTypes.DEFAULT_TYPE) -> None:
+    if not update.effective_chat:
+        return
+    chat_id = str(update.effective_chat.id)
+    db = SessionLocal()
+    try:
+        e = (
+            db.query(Employee)
+            .options(joinedload(Employee.projects))
+            .filter(Employee.tg_chat_id == chat_id)
+            .first()
+        )
+        if not e:
+            await update.message.reply_text("Сначала /start.")
+            return
+        if not e.projects:
+            await update.message.reply_text("У вас нет проектов.")
+            return
+        ids = [p.id for p in e.projects]
+    finally:
+        db.close()
+
+    await update.message.reply_text(f"⏳ Запускаю проверку {len(ids)} проектов…")
+    total_changes = 0
+    for pid in ids:
+        try:
+            total_changes += await asyncio.to_thread(run_check_for_project, pid)
+        except Exception as ex:
+            logger.exception("check failed for %s: %s", pid, ex)
+    await update.message.reply_text(f"✅ Готово. Изменений: {total_changes}")
+
+
+def main() -> None:
+    if not settings.tg_bot_token:
+        raise SystemExit("TG_BOT_TOKEN не задан в .env")
+    init_db()
+    app = Application.builder().token(settings.tg_bot_token).build()
+    app.add_handler(CommandHandler("start", cmd_start))
+    app.add_handler(CommandHandler("whoami", cmd_whoami))
+    app.add_handler(CommandHandler("list", cmd_list))
+    app.add_handler(CommandHandler("check", cmd_check))
+    logger.info("Bot polling…")
+    app.run_polling(allowed_updates=Update.ALL_TYPES)
+
+
+if __name__ == "__main__":
+    main()

app/config.py

@@ -0,0 +1,49 @@
+from pathlib import Path
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+BASE_DIR = Path(__file__).resolve().parent.parent
+DATA_DIR = BASE_DIR / "data"
+DATA_DIR.mkdir(exist_ok=True)
+
+
+def _resolve_sqlite_url(url: str) -> str:
+    """SQLAlchemy SQLite URL with a relative path is interpreted against CWD,
+    which breaks when the app is started from any directory other than the
+    project root. Anchor relative SQLite paths to BASE_DIR."""
+    prefix = "sqlite:///"
+    if not url.startswith(prefix):
+        return url
+    path_part = url[len(prefix):]
+    p = Path(path_part)
+    if p.is_absolute():
+        return url
+    resolved = (BASE_DIR / p).resolve()
+    resolved.parent.mkdir(parents=True, exist_ok=True)
+    return f"{prefix}{resolved}"
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_file=BASE_DIR / ".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    tg_bot_token: str = ""
+    web_host: str = "127.0.0.1"
+    web_port: int = 8000
+    public_base_path: str = ""
+    scrape_interval_hours: int = 4
+    database_url: str = f"sqlite:///{DATA_DIR / 'monitor.db'}"
+    admin_chat_id: str = ""
+    # Shared PIN that unlocks destructive web actions (delete/edit). Empty = gate
+    # disabled (everything allowed). See app/auth.py.
+    admin_pin: str = ""
+
+    def model_post_init(self, __context) -> None:
+        self.database_url = _resolve_sqlite_url(self.database_url)
+
+
+settings = Settings()

app/db.py

@@ -0,0 +1,31 @@
+from sqlalchemy import create_engine
+from sqlalchemy.orm import DeclarativeBase, sessionmaker
+
+from app.config import settings
+
+
+engine = create_engine(
+    settings.database_url,
+    connect_args={"check_same_thread": False} if settings.database_url.startswith("sqlite") else {},
+    future=True,
+)
+
+SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, future=True)
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()
+
+
+def init_db():
+    from app import models  # noqa: F401 — registers models on Base
+
+    Base.metadata.create_all(bind=engine)

app/models.py

@@ -0,0 +1,103 @@
+from datetime import datetime
+from enum import Enum
+
+from sqlalchemy import DateTime, Enum as SAEnum, Float, ForeignKey, Integer, String, Text, UniqueConstraint
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from app.db import Base
+
+
+class DealType(str, Enum):
+    SALE = "sale"
+    RENT = "rent"
+
+
+class Source(str, Enum):
+    PROPERTYFINDER = "propertyfinder"
+    BAYUT = "bayut"
+
+
+class ListingStatus(str, Enum):
+    ACTIVE = "active"
+    REMOVED = "removed"
+
+
+class Employee(Base):
+    __tablename__ = "employees"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    name: Mapped[str] = mapped_column(String(200))
+    tg_chat_id: Mapped[str | None] = mapped_column(String(64), unique=True, nullable=True)
+    tg_username: Mapped[str | None] = mapped_column(String(200), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
+
+    projects: Mapped[list["Project"]] = relationship(back_populates="owner")
+
+
+class Project(Base):
+    """Наш проект — квартира, которую агентство рекламирует."""
+
+    __tablename__ = "projects"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    title: Mapped[str] = mapped_column(String(300))
+    deal_type: Mapped[DealType] = mapped_column(SAEnum(DealType))
+    our_price: Mapped[float | None] = mapped_column(Float, nullable=True)
+    notes: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+    # Опциональные параметры — используются для подсказок похожих объявлений
+    dld_permit: Mapped[str | None] = mapped_column(String(100), index=True, nullable=True)
+    building: Mapped[str | None] = mapped_column(String(300), nullable=True)
+    bedrooms: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    size_sqft: Mapped[float | None] = mapped_column(Float, nullable=True)
+    our_url: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+    owner_id: Mapped[int] = mapped_column(ForeignKey("employees.id"))
+    owner: Mapped[Employee] = relationship(back_populates="projects")
+
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
+    last_checked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+
+    listings: Mapped[list["CompetitorListing"]] = relationship(
+        back_populates="project", cascade="all, delete-orphan"
+    )
+
+
+class CompetitorListing(Base):
+    """Объявление конкурента, найденное на PF/Bayut по DLD permit нашего проекта."""
+
+    __tablename__ = "competitor_listings"
+    __table_args__ = (UniqueConstraint("project_id", "source", "external_id", name="uq_listing"),)
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    project_id: Mapped[int] = mapped_column(ForeignKey("projects.id"))
+    project: Mapped[Project] = relationship(back_populates="listings")
+
+    source: Mapped[Source] = mapped_column(SAEnum(Source))
+    external_id: Mapped[str] = mapped_column(String(100))  # ID на стороне PF/Bayut
+    url: Mapped[str] = mapped_column(Text)
+    title: Mapped[str | None] = mapped_column(String(500), nullable=True)
+    agent_name: Mapped[str | None] = mapped_column(String(300), nullable=True)
+    agency_name: Mapped[str | None] = mapped_column(String(300), nullable=True)
+
+    current_price: Mapped[float | None] = mapped_column(Float, nullable=True)
+    currency: Mapped[str | None] = mapped_column(String(10), nullable=True, default="AED")
+    status: Mapped[ListingStatus] = mapped_column(SAEnum(ListingStatus), default=ListingStatus.ACTIVE)
+
+    first_seen_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
+    last_seen_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
+
+    price_history: Mapped[list["PriceHistory"]] = relationship(
+        back_populates="listing", cascade="all, delete-orphan", order_by="PriceHistory.recorded_at.desc()"
+    )
+
+
+class PriceHistory(Base):
+    __tablename__ = "price_history"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    listing_id: Mapped[int] = mapped_column(ForeignKey("competitor_listings.id"))
+    listing: Mapped[CompetitorListing] = relationship(back_populates="price_history")
+
+    price: Mapped[float | None] = mapped_column(Float, nullable=True)
+    recorded_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)

app/scheduler.py

@@ -0,0 +1,57 @@
+"""Background scheduler — runs run_check_all() every N hours.
+
+Run as a separate process: `python -m app.scheduler`.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+
+from apscheduler.schedulers.blocking import BlockingScheduler
+from apscheduler.triggers.interval import IntervalTrigger
+
+from app.config import settings
+from app.db import init_db
+from app.services.monitor import run_check_all
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+logger = logging.getLogger(__name__)
+
+
+def job() -> None:
+    logger.info("Scheduled scan starting…")
+    start = time.time()
+    summary = run_check_all()
+    elapsed = time.time() - start
+    total_changes = sum(c for c in summary.values() if c > 0)
+    logger.info(
+        "Scan done in %.1fs. Projects: %d, total changes: %d",
+        elapsed, len(summary), total_changes,
+    )
+
+
+def main() -> None:
+    init_db()
+    hours = max(1, settings.scrape_interval_hours)
+    scheduler = BlockingScheduler(timezone="UTC")
+    scheduler.add_job(
+        job,
+        trigger=IntervalTrigger(hours=hours),
+        # Omit next_run_time so APScheduler defaults the first run to now+interval
+        # (i.e. don't fire immediately at startup, fire after one interval, then
+        # every interval). Passing next_run_time=None instead creates the job in a
+        # PAUSED state and it never fires — that was the bug.
+        id="periodic-scan",
+        max_instances=1,
+        coalesce=True,
+    )
+    logger.info("Scheduler started — interval %d hour(s).", hours)
+    try:
+        scheduler.start()
+    except (KeyboardInterrupt, SystemExit):
+        logger.info("Scheduler stopped.")
+
+
+if __name__ == "__main__":
+    main()

app/scrapers/__init__.py

@@ -0,0 +1,5 @@
+from app.scrapers.base import ScrapedListing
+from app.scrapers.bayut import BayutScraper
+from app.scrapers.propertyfinder import PropertyFinderScraper
+
+__all__ = ["ScrapedListing", "BayutScraper", "PropertyFinderScraper"]

app/scrapers/base.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import json
+import logging
+import re
+from dataclasses import dataclass
+
+import httpx
+from bs4 import BeautifulSoup
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/131.0.0.0 Safari/537.36"
+    ),
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate, br",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+}
+
+
+@dataclass
+class ScrapedListing:
+    source: str            # "propertyfinder" | "bayut"
+    external_id: str       # listing id on the source
+    url: str
+    title: str | None
+    price: float | None
+    currency: str | None
+    permit_number: str | None
+    agent_name: str | None
+    agency_name: str | None
+    is_active: bool = True
+
+
+class ScraperError(Exception):
+    pass
+
+
+def fetch_html(url: str, timeout: float = 30.0) -> str:
+    """GET a URL with browser-like headers. Raises ScraperError on non-2xx."""
+    try:
+        with httpx.Client(headers=DEFAULT_HEADERS, follow_redirects=True, timeout=timeout) as client:
+            r = client.get(url)
+        if r.status_code in (403, 429):
+            raise ScraperError(f"Blocked by site ({r.status_code}) at {url}")
+        if r.status_code == 404:
+            return ""
+        r.raise_for_status()
+        return r.text
+    except httpx.HTTPError as e:
+        raise ScraperError(f"HTTP error for {url}: {e}") from e
+
+
+_NEXT_DATA_RE = re.compile(
+    r'<script[^>]+id="__NEXT_DATA__"[^>]*>(.*?)</script>',
+    re.DOTALL,
+)
+
+
+def extract_next_data(html: str) -> dict | None:
+    """Extract Next.js __NEXT_DATA__ JSON blob — both PF and Bayut are Next.js apps."""
+    if not html:
+        return None
+    m = _NEXT_DATA_RE.search(html)
+    if not m:
+        # Fallback via BeautifulSoup if regex misses (rare).
+        soup = BeautifulSoup(html, "lxml")
+        tag = soup.find("script", id="__NEXT_DATA__")
+        if not tag or not tag.string:
+            return None
+        raw = tag.string
+    else:
+        raw = m.group(1)
+    try:
+        return json.loads(raw)
+    except json.JSONDecodeError as e:
+        logger.warning("Failed to parse __NEXT_DATA__: %s", e)
+        return None
+
+
+def parse_price(value) -> float | None:
+    if value is None:
+        return None
+    if isinstance(value, (int, float)):
+        return float(value)
+    s = re.sub(r"[^\d.]", "", str(value))
+    try:
+        return float(s) if s else None
+    except ValueError:
+        return None

app/scrapers/bayut.py

@@ -0,0 +1,212 @@
+"""Bayut.com scraper.
+
+Two operations:
+- fetch_listing(url): read a listing detail page → ScrapedListing.
+- search_similar(building, bedrooms, deal_type): search Bayut for similar candidates.
+
+Bayut is a Next.js app; __NEXT_DATA__ contains the property in pageProps.
+Unlike PF, Bayut shows the permit number as text in the JSON.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from urllib.parse import quote_plus, urljoin
+
+from app.scrapers.base import (
+    ScrapedListing,
+    ScraperError,
+    extract_next_data,
+    fetch_html,
+    parse_price,
+)
+
+logger = logging.getLogger(__name__)
+
+BASE_URL = "https://www.bayut.com"
+SOURCE = "bayut"
+
+
+def _path_for_deal(deal_type: str) -> str:
+    return "to-buy" if deal_type == "sale" else "to-rent"
+
+
+def _walk(node):
+    if isinstance(node, dict):
+        yield node
+        for v in node.values():
+            yield from _walk(v)
+    elif isinstance(node, list):
+        for it in node:
+            yield from _walk(it)
+
+
+def _extract_price(item: dict) -> tuple[float | None, str | None]:
+    price = item.get("price")
+    if isinstance(price, dict):
+        val = price.get("value") or price.get("amount")
+        cur = price.get("currency") or "AED"
+        return parse_price(val), cur
+    if isinstance(price, (int, float, str)):
+        return parse_price(price), "AED"
+    return None, "AED"
+
+
+def _extract_broker(item: dict) -> tuple[str | None, str | None]:
+    agency = item.get("agency") or {}
+    agency_name = agency.get("name") if isinstance(agency, dict) else None
+    agent_name = item.get("contactName") or item.get("agentName") or item.get("ownerAgent", {}).get("name") if isinstance(item.get("ownerAgent"), dict) else item.get("contactName")
+    return agent_name, agency_name
+
+
+def _extract_permit(item: dict) -> str | None:
+    for key in ("permitNumber", "permit_number", "rera", "trakheesi", "permit"):
+        v = item.get(key)
+        if v:
+            return str(v).strip()
+    return None
+
+
+_ID_FROM_URL = re.compile(r"details-(\d+)\.html(?:[?#].*)?$")
+
+
+def _extract_id_from_url(url: str) -> str | None:
+    m = _ID_FROM_URL.search(url)
+    return m.group(1) if m else None
+
+
+def _is_listing_dict(item: dict) -> bool:
+    if not isinstance(item, dict):
+        return False
+    has_price = "price" in item
+    has_id = any(k in item for k in ("externalID", "id", "objectID"))
+    return has_price and has_id
+
+
+class BayutScraper:
+    source = SOURCE
+
+    def fetch_listing(self, url: str) -> ScrapedListing | None:
+        try:
+            html = fetch_html(url)
+        except ScraperError as e:
+            logger.warning("Bayut refetch failed for %s: %s", url, e)
+            return None
+
+        if not html:
+            return ScrapedListing(
+                source=SOURCE, external_id=_extract_id_from_url(url) or "", url=url,
+                title=None, price=None, currency=None, permit_number=None,
+                agent_name=None, agency_name=None, is_active=False,
+            )
+
+        data = extract_next_data(html)
+        if not data:
+            return None
+
+        best = None
+        best_score = -1
+        for node in _walk(data):
+            if not _is_listing_dict(node):
+                continue
+            score = 0
+            if "title" in node or "name" in node:
+                score += 2
+            if "agency" in node or "contactName" in node:
+                score += 2
+            if "rooms" in node or "bedrooms" in node:
+                score += 1
+            if score > best_score:
+                best_score = score
+                best = node
+
+        if best is None:
+            logger.warning("Bayut: no listing dict found in __NEXT_DATA__ for %s", url)
+            return None
+
+        price, currency = _extract_price(best)
+        agent_name, agency_name = _extract_broker(best)
+        ext_id = (
+            str(best.get("externalID") or best.get("id") or "")
+            or _extract_id_from_url(url)
+            or ""
+        )
+        return ScrapedListing(
+            source=SOURCE,
+            external_id=ext_id,
+            url=url,
+            title=best.get("title") or best.get("name"),
+            price=price,
+            currency=currency,
+            permit_number=_extract_permit(best),
+            agent_name=agent_name,
+            agency_name=agency_name,
+            is_active=True,
+        )
+
+    def search_similar(
+        self,
+        building: str | None,
+        bedrooms: int | None,
+        deal_type: str,
+        limit: int = 20,
+        location_url: str | None = None,
+    ) -> list[ScrapedListing]:
+        if not building:
+            return []
+        path = _path_for_deal(deal_type)
+        q = quote_plus(building.strip())
+        url = f"{BASE_URL}/{path}/property/dubai/?q={q}"
+        if bedrooms is not None:
+            url += f"&beds_in={bedrooms}"
+        logger.info("Bayut search_similar: %s", url)
+
+        try:
+            html = fetch_html(url)
+        except ScraperError as e:
+            logger.warning("Bayut search failed: %s", e)
+            return []
+
+        data = extract_next_data(html)
+        if not data:
+            return []
+
+        results: list[ScrapedListing] = []
+        seen_ids: set[str] = set()
+        for node in _walk(data):
+            if not _is_listing_dict(node):
+                continue
+            ext_id = str(node.get("externalID") or node.get("id") or "")
+            if not ext_id or ext_id in seen_ids:
+                continue
+
+            title = node.get("title") or node.get("name") or ""
+            if building.lower() not in (title or "").lower():
+                slug = str(node.get("slug") or "").lower()
+                building_token = building.lower().replace(" ", "-")
+                if building_token not in slug:
+                    continue
+
+            seen_ids.add(ext_id)
+            price, currency = _extract_price(node)
+            agent_name, agency_name = _extract_broker(node)
+            cand_url = urljoin(BASE_URL, f"/property/details-{ext_id}.html")
+
+            results.append(
+                ScrapedListing(
+                    source=SOURCE,
+                    external_id=ext_id,
+                    url=cand_url,
+                    title=title or None,
+                    price=price,
+                    currency=currency,
+                    permit_number=_extract_permit(node),
+                    agent_name=agent_name,
+                    agency_name=agency_name,
+                    is_active=True,
+                )
+            )
+            if len(results) >= limit:
+                break
+        return results

app/scrapers/propertyfinder.py

@@ -0,0 +1,325 @@
+"""PropertyFinder.ae scraper.
+
+Two operations:
+- fetch_listing(url): read a listing detail page → ScrapedListing (title/price/agent/permit).
+- search_similar(building, bedrooms, deal_type): search PF for similar candidates
+  by building name + bedrooms filter → list[ScrapedListing].
+
+PF is a Next.js app — listing data sits in <script id="__NEXT_DATA__">.
+Note: PF intentionally hides the Trakheesi permit as an image on the detail page,
+so permit may come back as None — that's fine, we don't depend on it.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from urllib.parse import urljoin
+
+from app.scrapers.base import (
+    ScrapedListing,
+    ScraperError,
+    extract_next_data,
+    fetch_html,
+    parse_price,
+)
+
+logger = logging.getLogger(__name__)
+
+BASE_URL = "https://www.propertyfinder.ae"
+SOURCE = "propertyfinder"
+
+# PF location hierarchy, most specific first. search_similar scopes by the most
+# specific id available on a reference listing page.
+_LOC_TYPE_PRIORITY = {
+    "TOWER": 5,
+    "BUILDING": 4,
+    "DEVELOPMENT": 3,
+    "SUBCOMMUNITY": 2,
+    "COMMUNITY": 1,
+    "CITY": 0,
+}
+
+
+def _category_for_deal(deal_type: str) -> int:
+    return 1 if deal_type == "sale" else 2
+
+
+def _get(d, *keys, default=None):
+    cur = d
+    for k in keys:
+        if not isinstance(cur, dict):
+            return default
+        cur = cur.get(k)
+        if cur is None:
+            return default
+    return cur
+
+
+def _walk(node):
+    """Iterate over every dict in a nested JSON structure."""
+    if isinstance(node, dict):
+        yield node
+        for v in node.values():
+            yield from _walk(v)
+    elif isinstance(node, list):
+        for it in node:
+            yield from _walk(it)
+
+
+def _extract_price(item: dict) -> tuple[float | None, str | None]:
+    price = item.get("price")
+    if isinstance(price, dict):
+        val = price.get("value") or price.get("amount") or price.get("min") or price.get("from")
+        cur = price.get("currency") or "AED"
+        return parse_price(val), cur
+    if isinstance(price, (int, float, str)):
+        return parse_price(price), item.get("currency") or "AED"
+    return None, "AED"
+
+
+def _extract_broker(item: dict) -> tuple[str | None, str | None]:
+    broker = item.get("broker") or item.get("agency") or {}
+    agent = item.get("agent") or item.get("contact") or {}
+    agency_name = broker.get("name") if isinstance(broker, dict) else None
+    agent_name = agent.get("name") if isinstance(agent, dict) else None
+    return agent_name, agency_name
+
+
+def _extract_permit(item: dict) -> str | None:
+    for key in ("permit_number", "permitNumber", "trakheesi", "rera", "permit"):
+        v = item.get(key)
+        if v:
+            return str(v).strip()
+    reg = item.get("regulatory") or item.get("regulation") or {}
+    if isinstance(reg, dict):
+        for key in ("permit", "permit_number", "trakheesi", "rera"):
+            v = reg.get(key)
+            if v:
+                return str(v).strip()
+    return None
+
+
+def _find_permit_on_page(data: dict) -> str | None:
+    """The DLD permit number lives in a regulatory block rendered as an image,
+    but its plain value is still in __NEXT_DATA__: the dict that carries a
+    `permit_validation_url` (the Trakheesi link) also has the number in
+    `number`. Walk the page and pull it out."""
+    for node in _walk(data):
+        if isinstance(node, dict) and node.get("permit_validation_url") and node.get("number"):
+            return str(node["number"]).strip()
+    return None
+
+
+_ID_FROM_URL = re.compile(r"-(\d+)\.html(?:[?#].*)?$")
+
+
+def _extract_id_from_url(url: str) -> str | None:
+    m = _ID_FROM_URL.search(url)
+    return m.group(1) if m else None
+
+
+def _is_listing_dict(item: dict) -> bool:
+    """Heuristic: a listing dict contains a price plus an id-like field."""
+    if not isinstance(item, dict):
+        return False
+    has_price = "price" in item
+    has_id = any(k in item for k in ("id", "reference", "listing_id", "externalID"))
+    return has_price and has_id
+
+
+class PropertyFinderScraper:
+    source = SOURCE
+
+    def fetch_listing(self, url: str) -> ScrapedListing | None:
+        """Refetch a known listing URL. Returns:
+        - ScrapedListing(is_active=False) if the URL returns 404 (listing removed)
+        - ScrapedListing with current data if alive
+        - None on network/parse failure (we won't update the DB in that case)
+        """
+        try:
+            html = fetch_html(url)
+        except ScraperError as e:
+            logger.warning("PF refetch failed for %s: %s", url, e)
+            return None
+
+        if not html:
+            return ScrapedListing(
+                source=SOURCE, external_id=_extract_id_from_url(url) or "", url=url,
+                title=None, price=None, currency=None, permit_number=None,
+                agent_name=None, agency_name=None, is_active=False,
+            )
+
+        data = extract_next_data(html)
+        if not data:
+            return None
+
+        # On a PF detail page the property dict is nested in pageProps. Walk and pick
+        # the dict that has both a "price" and an id, ignoring trivial nested ones.
+        best = None
+        best_score = -1
+        for node in _walk(data):
+            if not _is_listing_dict(node):
+                continue
+            score = 0
+            if "title" in node or "name" in node:
+                score += 2
+            if any(k in node for k in ("broker", "agent", "agency")):
+                score += 2
+            if "bedrooms" in node or "rooms" in node:
+                score += 1
+            if score > best_score:
+                best_score = score
+                best = node
+
+        if best is None:
+            logger.warning("PF: no listing dict found in __NEXT_DATA__ for %s", url)
+            return None
+
+        price, currency = _extract_price(best)
+        agent_name, agency_name = _extract_broker(best)
+        ext_id = (
+            str(best.get("id") or best.get("reference") or best.get("listing_id") or "")
+            or _extract_id_from_url(url)
+            or ""
+        )
+        return ScrapedListing(
+            source=SOURCE,
+            external_id=ext_id,
+            url=url,
+            title=best.get("title") or best.get("name"),
+            price=price,
+            currency=currency,
+            permit_number=_find_permit_on_page(data) or _extract_permit(best),
+            agent_name=agent_name,
+            agency_name=agency_name,
+            is_active=True,
+        )
+
+    def get_permit(self, url: str) -> str | None:
+        """Fetch a listing page and return only its DLD permit number (or None).
+        Used to compare candidates against our own permit during suggestions."""
+        try:
+            html = fetch_html(url)
+        except ScraperError as e:
+            logger.warning("PF get_permit fetch failed for %s: %s", url, e)
+            return None
+        data = extract_next_data(html)
+        return _find_permit_on_page(data) if data else None
+
+    def resolve_location_id(self, listing_url: str) -> int | None:
+        """Read a PF listing page and return the most specific location id
+        (tower > building > subcommunity > community).
+
+        PF's search only filters by numeric location id (`l=`); the free-text
+        `q=` param does NOT scope results to a building — it returns unrelated
+        recommendations. So we derive the location id from a known listing that
+        sits in the same building (our own listing, or an already-tracked one).
+        """
+        try:
+            html = fetch_html(listing_url)
+        except ScraperError as e:
+            logger.warning("PF resolve_location_id fetch failed for %s: %s", listing_url, e)
+            return None
+        data = extract_next_data(html)
+        if not data:
+            return None
+
+        best_id: object = None
+        best_rank = -1
+        for node in _walk(data):
+            if not isinstance(node, dict):
+                continue
+            rank = _LOC_TYPE_PRIORITY.get(str(node.get("type", "")).upper(), -1)
+            if rank > best_rank and node.get("id") and node.get("name"):
+                best_rank, best_id = rank, node.get("id")
+        try:
+            return int(best_id) if best_id is not None else None
+        except (TypeError, ValueError):
+            return None
+
+    def search_similar(
+        self,
+        building: str | None,
+        bedrooms: int | None,
+        deal_type: str,
+        limit: int = 200,
+        location_url: str | None = None,
+        max_pages: int = 8,
+    ) -> list[ScrapedListing]:
+        """Search PF for candidates in the same building, scoped by location id.
+
+        `location_url` is a reference listing in the target building (our own
+        listing or an already-tracked competitor) — we resolve it to a PF
+        location id and search by `l=`. Without it we can't reliably scope a
+        building search on PF, so we return nothing rather than garbage.
+
+        Paginates: a same-permit competitor can sit on any results page (PF
+        can't be queried by permit), so we collect across pages up to
+        `max_pages`/`limit`.
+        """
+        location_id = self.resolve_location_id(location_url) if location_url else None
+        if location_id is None:
+            logger.info(
+                "PF search_similar: no location id (url=%r) — skipping (q= text search "
+                "does not filter by building on PF)", location_url,
+            )
+            return []
+
+        c = _category_for_deal(deal_type)
+        base = f"{BASE_URL}/en/search?c={c}&l={location_id}"
+        if bedrooms is not None:
+            base += f"&bf={bedrooms}&bt={bedrooms}"  # PF uses bf=bedrooms-from, bt=bedrooms-to
+
+        results: list[ScrapedListing] = []
+        seen_ids: set[str] = set()
+        for page in range(1, max_pages + 1):
+            page_url = base if page == 1 else f"{base}&page={page}"
+            try:
+                html = fetch_html(page_url)
+            except ScraperError as e:
+                logger.warning("PF search failed (page %d): %s", page, e)
+                break
+            data = extract_next_data(html)
+            if not data:
+                break
+
+            new_on_page = 0
+            for node in _walk(data):
+                if not _is_listing_dict(node):
+                    continue
+                ext_id = str(node.get("id") or node.get("reference") or "")
+                if not ext_id or ext_id in seen_ids:
+                    continue
+                seen_ids.add(ext_id)
+                new_on_page += 1
+
+                # Results are scoped to the location by l=, so no title filter.
+                title = node.get("title") or node.get("name") or ""
+                price, currency = _extract_price(node)
+                agent_name, agency_name = _extract_broker(node)
+                share = node.get("share_url") or node.get("path")
+                cand_url = share if str(share).startswith("http") else urljoin(BASE_URL, str(share or ""))
+
+                results.append(
+                    ScrapedListing(
+                        source=SOURCE,
+                        external_id=ext_id,
+                        url=cand_url or page_url,
+                        title=title or None,
+                        price=price,
+                        currency=currency,
+                        permit_number=_extract_permit(node),
+                        agent_name=agent_name,
+                        agency_name=agency_name,
+                        is_active=True,
+                    )
+                )
+                if len(results) >= limit:
+                    break
+
+            # No new listings on this page → we've passed the last page.
+            if len(results) >= limit or new_on_page == 0:
+                break
+        logger.info("PF search_similar: collected %d candidates (l=%s)", len(results), location_id)
+        return results

app/services/monitor.py

@@ -0,0 +1,354 @@
+"""Core monitoring logic.
+
+Per project, for every CompetitorListing we already track:
+  1. Re-fetch the listing's URL.
+  2. Detect:
+     - Price change → 📈📉 notify
+     - URL returns 404 / removed → ❌ notify, mark removed
+     - Reappeared after removal → ♻️ notify
+  3. Snapshot every price into PriceHistory.
+
+Adding new competitors is done via the web UI (user pastes URLs) — not here.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
+
+from sqlalchemy.orm import Session
+
+from app.db import SessionLocal
+from app.models import (
+    CompetitorListing,
+    Employee,
+    ListingStatus,
+    PriceHistory,
+    Project,
+    Source,
+)
+from app.scrapers import BayutScraper, PropertyFinderScraper, ScrapedListing
+from app.services.notifier import notify_admin, send_message
+
+logger = logging.getLogger(__name__)
+
+PF = PropertyFinderScraper()
+BAYUT = BayutScraper()
+
+# Same-building suggestions beyond exact permit matches are a browse heuristic —
+# cap how many we show so the page stays usable.
+_SUGGEST_OTHERS_LIMIT = 30
+
+# Bayut moved to fully client-side rendering (no __NEXT_DATA__, Algolia keys
+# hidden), so it can't be scraped over plain HTTP — disabled until we add a
+# headless-browser fetcher. Flip to True once that exists.
+BAYUT_ENABLED = False
+
+
+def _scraper_for(source: Source):
+    return PF if source == Source.PROPERTYFINDER else BAYUT
+
+
+def detect_source_from_url(url: str) -> Source | None:
+    u = (url or "").lower()
+    if "propertyfinder" in u:
+        return Source.PROPERTYFINDER
+    if "bayut.com" in u:
+        return Source.BAYUT
+    return None
+
+
+def _fmt_price(value: float | None, currency: str | None = "AED") -> str:
+    if value is None:
+        return "—"
+    return f"{value:,.0f} {currency or 'AED'}".replace(",", " ")
+
+
+def _source_label(source: str) -> str:
+    return {"propertyfinder": "PropertyFinder", "bayut": "Bayut"}.get(source, source)
+
+
+def add_competitor_url(db: Session, project: Project, url: str) -> tuple[CompetitorListing | None, str]:
+    """User-facing entrypoint: paste a URL → create CompetitorListing for the project.
+    Returns (listing, error_message). error_message is empty on success.
+    """
+    url = (url or "").strip()
+    if not url:
+        return None, "URL пустой"
+    source = detect_source_from_url(url)
+    if source is None:
+        return None, "URL должен быть с propertyfinder.ae или bayut.com"
+    if source == Source.BAYUT and not BAYUT_ENABLED:
+        return None, (
+            "Bayut временно не поддерживается — площадка перешла на защищённый "
+            "рендеринг. Добавляйте ссылки PropertyFinder."
+        )
+
+    scraper = _scraper_for(source)
+    scraped = scraper.fetch_listing(url)
+    if scraped is None:
+        return None, "Не удалось загрузить страницу — сайт мог заблокировать запрос, попробуйте позже"
+    if not scraped.is_active:
+        return None, "Страница объявления вернула 404 — ссылка битая или объявление снято"
+
+    ext_id = scraped.external_id or url  # fallback if id wasn't found
+    existing = (
+        db.query(CompetitorListing)
+        .filter(
+            CompetitorListing.project_id == project.id,
+            CompetitorListing.source == source,
+            CompetitorListing.external_id == ext_id,
+        )
+        .first()
+    )
+    if existing:
+        return None, "Это объявление уже добавлено в проект"
+
+    now = datetime.utcnow()
+    listing = CompetitorListing(
+        project_id=project.id,
+        source=source,
+        external_id=ext_id,
+        url=url,
+        title=scraped.title,
+        agent_name=scraped.agent_name,
+        agency_name=scraped.agency_name,
+        current_price=scraped.price,
+        currency=scraped.currency or "AED",
+        status=ListingStatus.ACTIVE,
+        first_seen_at=now,
+        last_seen_at=now,
+    )
+    db.add(listing)
+    db.flush()
+    if scraped.price is not None:
+        db.add(PriceHistory(listing_id=listing.id, price=scraped.price, recorded_at=now))
+    db.commit()
+    return listing, ""
+
+
+def add_competitor_urls(db: Session, project: Project, urls: list[str]) -> dict:
+    """Add several pasted/selected URLs in one go (used by the suggest page's
+    multi-select). Processes them sequentially — each one re-fetches the page —
+    and reports a summary. Returns {'added': int, 'skipped': int, 'errors': [..]}.
+    """
+    added = 0
+    skipped = 0
+    errors: list[str] = []
+    seen: set[str] = set()
+    for raw in urls:
+        url = (raw or "").strip()
+        if not url or url in seen:
+            continue
+        seen.add(url)
+        listing, err = add_competitor_url(db, project, url)
+        if err == "Это объявление уже добавлено в проект":
+            skipped += 1
+        elif err:
+            errors.append(err)
+        else:
+            added += 1
+    return {"added": added, "skipped": skipped, "errors": errors}
+
+
+def check_project(db: Session, project: Project) -> list[str]:
+    """Re-scan all tracked competitor listings for one project. Returns notification texts."""
+    changes: list[str] = []
+    now = datetime.utcnow()
+
+    for listing in list(project.listings):
+        scraper = _scraper_for(listing.source)
+        scraped = scraper.fetch_listing(listing.url)
+        if scraped is None:
+            # Network/parse failure — skip without changing state, try again next cycle.
+            continue
+
+        if not scraped.is_active:
+            if listing.status == ListingStatus.ACTIVE:
+                listing.status = ListingStatus.REMOVED
+                changes.append(
+                    f"❌ <b>Объявление удалено</b> — {_source_label(listing.source.value)}\n"
+                    f"{listing.title or 'без названия'}\n"
+                    f"Последняя цена: {_fmt_price(listing.current_price, listing.currency)}\n"
+                    f"{listing.url}"
+                )
+            continue
+
+        listing.last_seen_at = now
+
+        if listing.status != ListingStatus.ACTIVE:
+            listing.status = ListingStatus.ACTIVE
+            changes.append(
+                f"♻️ <b>Объявление снова активно</b> — {_source_label(listing.source.value)}\n"
+                f"{listing.title or 'без названия'}\n{listing.url}"
+            )
+
+        # Update metadata that may have changed
+        if scraped.title:
+            listing.title = scraped.title
+        if scraped.agent_name:
+            listing.agent_name = scraped.agent_name
+        if scraped.agency_name:
+            listing.agency_name = scraped.agency_name
+
+        old_price = listing.current_price
+        new_price = scraped.price
+        if new_price is not None and old_price is not None and new_price != old_price:
+            delta = new_price - old_price
+            pct = (delta / old_price * 100.0) if old_price else 0.0
+            arrow = "📈" if delta > 0 else "📉"
+            changes.append(
+                f"{arrow} <b>Цена изменилась</b> — {_source_label(listing.source.value)}\n"
+                f"{listing.title or 'без названия'}\n"
+                f"Было: {_fmt_price(old_price, listing.currency)}\n"
+                f"Стало: {_fmt_price(new_price, scraped.currency or listing.currency)} "
+                f"({'+' if delta > 0 else ''}{delta:,.0f} / {pct:+.1f}%)\n"
+                f"{listing.url}".replace(",", " ")
+            )
+            listing.current_price = new_price
+            listing.currency = scraped.currency or listing.currency
+            db.add(PriceHistory(listing_id=listing.id, price=new_price, recorded_at=now))
+        elif new_price is not None and old_price is None:
+            listing.current_price = new_price
+            listing.currency = scraped.currency or listing.currency
+            db.add(PriceHistory(listing_id=listing.id, price=new_price, recorded_at=now))
+
+    project.last_checked_at = now
+    db.commit()
+    return changes
+
+
+def _notify_owner(project: Project, changes: list[str]) -> None:
+    if not changes:
+        return
+    owner: Employee | None = project.owner
+    if not owner or not owner.tg_chat_id:
+        logger.warning("Project %s has no owner chat_id; skipping notification", project.id)
+        notify_admin(
+            f"⚠️ Проект <b>{project.title}</b> (#{project.id}) — {len(changes)} изменений, "
+            f"но у владельца не задан tg_chat_id."
+        )
+        return
+
+    header = (
+        f"🏠 <b>{project.title}</b>\n"
+        f"Тип: {project.deal_type.value} · Изменений: {len(changes)}\n"
+        f"——————————"
+    )
+    send_message(owner.tg_chat_id, header)
+    for c in changes:
+        send_message(owner.tg_chat_id, c)
+
+
+def _reference_url(project: Project, source: Source) -> str | None:
+    """A known listing URL in the project's building for the given source.
+
+    Portals (PF) scope a building search by an internal numeric location id, not
+    by free text — so we hand the scraper a real listing in the same building
+    (our own `our_url`, else an already-tracked competitor) to resolve that id.
+    """
+    candidates: list[str] = []
+    if project.our_url:
+        candidates.append(project.our_url)
+    candidates.extend(l.url for l in project.listings if l.source == source)
+    for url in candidates:
+        if detect_source_from_url(url) == source:
+            return url
+    return None
+
+
+def resolve_our_permit(project: Project) -> str | None:
+    """Our project's DLD permit number. Prefer the value the user typed; else
+    read it off our own listing (PF exposes the number in __NEXT_DATA__)."""
+    if project.dld_permit and project.dld_permit.strip():
+        return project.dld_permit.strip()
+    ref = _reference_url(project, Source.PROPERTYFINDER)
+    return PF.get_permit(ref) if ref else None
+
+
+def suggest_similar(project: Project, our_permit: str | None = None) -> dict[str, list[ScrapedListing]]:
+    """Search PF + Bayut for listings in this project's building.
+
+    Candidates that share our DLD permit are the same physical listing under a
+    different broker (rare, but it happens) — those are surfaced first. The rest
+    are same-building heuristics. Returns {'propertyfinder': [...], 'bayut': [...]}.
+    """
+    out: dict[str, list[ScrapedListing]] = {"propertyfinder": [], "bayut": []}
+    bedrooms = project.bedrooms
+    deal_type = project.deal_type.value
+
+    try:
+        out["propertyfinder"] = PF.search_similar(
+            project.building, bedrooms, deal_type,
+            location_url=_reference_url(project, Source.PROPERTYFINDER),
+        )
+    except Exception as e:
+        logger.exception("PF suggest failed: %s", e)
+    if BAYUT_ENABLED:
+        try:
+            out["bayut"] = BAYUT.search_similar(
+                project.building, bedrooms, deal_type,
+                location_url=_reference_url(project, Source.BAYUT),
+            )
+        except Exception as e:
+            logger.exception("Bayut suggest failed: %s", e)
+
+    # Hide candidates already tracked for this project — and our own listing.
+    excluded = {(l.source.value, l.external_id) for l in project.listings}
+    if project.our_url:
+        own_src = detect_source_from_url(project.our_url)
+        m = re.search(r"(\d+)\.html", project.our_url)
+        if own_src and m:
+            excluded.add((own_src.value, m.group(1)))
+    for src in out:
+        out[src] = [c for c in out[src] if (src, c.external_id) not in excluded]
+
+    # Permit-first: PF can't be queried by permit and search results don't carry
+    # it — so read each PF candidate's permit (concurrently) and put the ones
+    # matching ours first. Keep all matches; cap the same-building remainder.
+    if our_permit and out["propertyfinder"]:
+        pf = out["propertyfinder"]
+        try:
+            with ThreadPoolExecutor(max_workers=12) as ex:
+                permits = list(ex.map(PF.get_permit, [c.url for c in pf]))
+            for cand, permit in zip(pf, permits):
+                cand.permit_number = permit
+            matches = [c for c in pf if c.permit_number == our_permit]
+            others = [c for c in pf if c.permit_number != our_permit]
+            out["propertyfinder"] = matches + others[:_SUGGEST_OTHERS_LIMIT]
+        except Exception as e:
+            logger.exception("PF permit enrichment failed: %s", e)
+
+    return out
+
+
+def run_check_for_project(project_id: int) -> int:
+    db = SessionLocal()
+    try:
+        project = db.get(Project, project_id)
+        if not project:
+            return 0
+        changes = check_project(db, project)
+        _notify_owner(project, changes)
+        return len(changes)
+    finally:
+        db.close()
+
+
+def run_check_all() -> dict[int, int]:
+    db = SessionLocal()
+    try:
+        ids = [p.id for p in db.query(Project).all()]
+    finally:
+        db.close()
+
+    summary = {}
+    for pid in ids:
+        try:
+            summary[pid] = run_check_for_project(pid)
+        except Exception as e:
+            logger.exception("Check for project %s failed: %s", pid, e)
+            summary[pid] = -1
+    return summary

app/services/notifier.py

@@ -0,0 +1,48 @@
+"""Telegram notification sender. Uses httpx directly — no bot framework needed
+for outbound messages, so we can call it from the scheduler thread without
+needing an event loop."""
+
+from __future__ import annotations
+
+import logging
+
+import httpx
+
+from app.config import settings
+
+logger = logging.getLogger(__name__)
+
+TG_API = "https://api.telegram.org"
+
+
+def send_message(chat_id: str, text: str, parse_mode: str = "HTML") -> bool:
+    if not settings.tg_bot_token:
+        logger.warning("TG_BOT_TOKEN not set — skipping notification to %s", chat_id)
+        return False
+    if not chat_id:
+        logger.warning("Empty chat_id — skipping notification")
+        return False
+    url = f"{TG_API}/bot{settings.tg_bot_token}/sendMessage"
+    try:
+        with httpx.Client(timeout=15.0) as client:
+            r = client.post(
+                url,
+                json={
+                    "chat_id": chat_id,
+                    "text": text,
+                    "parse_mode": parse_mode,
+                    "disable_web_page_preview": False,
+                },
+            )
+        if r.status_code != 200:
+            logger.error("TG send failed: %s %s", r.status_code, r.text)
+            return False
+        return True
+    except httpx.HTTPError as e:
+        logger.error("TG send exception: %s", e)
+        return False
+
+
+def notify_admin(text: str) -> None:
+    if settings.admin_chat_id:
+        send_message(settings.admin_chat_id, text)

app/templates/admin_login.html

@@ -0,0 +1,29 @@
+{% extends "base.html" %}
+{% block title %}Вход администратора — DLD Monitor{% endblock %}
+{% block content %}
+
+<div class="row justify-content-center">
+  <div class="col-md-5">
+    <div class="card shadow-sm">
+      <div class="card-body">
+        <h5 class="mb-3">🔒 Режим администратора</h5>
+        <p class="text-muted small">
+          Введите PIN, чтобы разблокировать удаление проектов, конкурентов и
+          редактирование/удаление сотрудников. Сессия — на 8 часов.
+        </p>
+        {% if error %}<div class="alert alert-danger py-2">{{ error }}</div>{% endif %}
+        <form method="post" action="{{ url_path('/admin/login') }}">
+          <input type="hidden" name="next" value="{{ next }}">
+          <div class="mb-3">
+            <input type="password" name="pin" required autofocus class="form-control"
+                   placeholder="PIN администратора">
+          </div>
+          <button class="btn btn-primary w-100">Войти</button>
+        </form>
+        <a href="{{ url_path(next) }}" class="btn btn-link btn-sm w-100 mt-2">← Отмена</a>
+      </div>
+    </div>
+  </div>
+</div>
+
+{% endblock %}

app/templates/base.html

@@ -0,0 +1,88 @@
+<!doctype html>
+<html lang="ru">
+<head>
+<meta charset="utf-8">
+<title>{% block title %}DLD Monitor{% endblock %}</title>
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet">
+<style>
+  body { background: #f4f5f9; }
+  .navbar-brand { letter-spacing: .3px; }
+  .nav-link.active { font-weight: 600; color: #0d6efd !important; }
+  .badge-sale { background: #198754; }
+  .badge-rent { background: #0d6efd; }
+  .card { border: 1px solid #e9ecef; }
+  .listing-card { border-left: 4px solid #dee2e6; }
+  .listing-card.removed { border-left-color: #dc3545; opacity: 0.65; }
+  .listing-card.active { border-left-color: #198754; }
+  .src-pf { color: #d63384; font-weight: 600; }
+  .src-bayut { color: #0d6efd; font-weight: 600; }
+  .price-up { color: #dc3545; }
+  .price-down { color: #198754; }
+  pre.permit { display: inline; background: #eee; padding: 2px 6px; border-radius: 4px; }
+  .stat-chip { background:#fff; border:1px solid #e9ecef; border-radius:.5rem; padding:.4rem .8rem; }
+  .stat-chip .n { font-size:1.25rem; font-weight:700; line-height:1; }
+  footer { color:#9aa0a6; font-size:.8rem; }
+  /* Mobile tweaks */
+  @media (max-width: 575.98px) {
+    body { background: #fff; }
+    .container { padding-left: .75rem; padding-right: .75rem; }
+    h3 { font-size: 1.35rem; }
+    .card-body { padding: .75rem; }
+    .fs-4 { font-size: 1.1rem !important; }
+  }
+</style>
+{% block head %}{% endblock %}
+</head>
+<body>
+<nav class="navbar navbar-expand-lg bg-white border-bottom mb-4">
+  <div class="container">
+    <a class="navbar-brand fw-bold" href="{{ url_path('/') }}">🏠 DLD Monitor</a>
+    <button class="navbar-toggler" type="button" data-bs-toggle="collapse"
+            data-bs-target="#mainnav" aria-controls="mainnav"
+            aria-expanded="false" aria-label="Меню">
+      <span class="navbar-toggler-icon"></span>
+    </button>
+    <div class="collapse navbar-collapse" id="mainnav">
+      <ul class="navbar-nav me-auto mb-2 mb-lg-0">
+        <li class="nav-item">
+          <a href="{{ url_path('/') }}" class="nav-link {{ 'active' if request.url.path == '/' or request.url.path.startswith('/projects') else '' }}">Проекты</a>
+        </li>
+        <li class="nav-item">
+          <a href="{{ url_path('/employees') }}" class="nav-link {{ 'active' if request.url.path.startswith('/employees') else '' }}">Сотрудники</a>
+        </li>
+      </ul>
+      <div class="d-flex flex-column flex-lg-row align-items-stretch align-items-lg-center gap-2">
+        <a href="{{ url_path('/projects/new') }}" class="btn btn-sm btn-primary">+ Проект</a>
+        <span class="vr d-none d-lg-block"></span>
+        {% if request.state.is_admin %}
+          {% if request.state.admin_configured %}
+            <span class="badge bg-success align-self-start align-self-lg-center">🔓 админ</span>
+            <form method="post" action="{{ url_path('/admin/logout') }}">
+              <input type="hidden" name="next" value="{{ request.url.path }}">
+              <button class="btn btn-sm btn-outline-secondary w-100">Выйти</button>
+            </form>
+          {% else %}
+            <span class="badge bg-warning text-dark align-self-start align-self-lg-center" title="ADMIN_PIN не задан в .env — права не ограничены">⚠ без PIN</span>
+          {% endif %}
+        {% else %}
+          <a href="{{ url_path('/admin/login?next=' ~ request.url.path) }}" class="btn btn-sm btn-outline-dark">🔒 Админ</a>
+        {% endif %}
+      </div>
+    </div>
+  </div>
+</nav>
+
+<div class="container">
+  {% if flash %}
+    <div class="alert alert-info">{{ flash }}</div>
+  {% endif %}
+  {% block content %}{% endblock %}
+</div>
+
+<footer class="container text-center py-4">DLD Monitor · HOME LIGA REAL ESTATE</footer>
+
+<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js"></script>
+{% block scripts %}{% endblock %}
+</body>
+</html>

app/templates/employees.html

@@ -0,0 +1,86 @@
+{% extends "base.html" %}
+{% block title %}Сотрудники — DLD Monitor{% endblock %}
+{% block content %}
+
+<h3>Сотрудники</h3>
+
+<div class="alert alert-info">
+  <b>Как подключить Telegram:</b>
+  <ol class="mb-0">
+    <li>Сотрудник пишет боту в Telegram команду <code>/whoami</code> — бот вернёт его <code>chat_id</code>.</li>
+    <li>Вставьте этот <code>chat_id</code> в поле справа от имени и нажмите «Сохранить».</li>
+  </ol>
+  Если сотрудник пока не знает свой chat_id — пусть просто отправит <code>/start</code>, бот ответит и пришлёт id.
+</div>
+
+<form method="post" action="{{ url_path('/employees') }}" class="row g-2 mb-4 bg-white p-3 rounded shadow-sm">
+  <div class="col-md-5">
+    <input name="name" required class="form-control" placeholder="Имя сотрудника">
+  </div>
+  <div class="col-md-4">
+    <input name="tg_username" class="form-control" placeholder="@username (опционально)">
+  </div>
+  <div class="col-md-3">
+    <button class="btn btn-primary w-100">Добавить</button>
+  </div>
+</form>
+
+{% if not request.state.is_admin %}
+  <div class="alert alert-secondary py-2 small">
+    🔒 Редактирование и удаление сотрудников доступно только в режиме админа.
+    <a href="{{ url_path('/admin/login?next=/employees') }}">Войти</a>.
+  </div>
+{% endif %}
+
+<div class="table-responsive">
+<table class="table bg-white rounded shadow-sm align-middle">
+  <thead class="table-light">
+    <tr><th>Имя</th><th>@username</th><th>Chat ID</th><th>Проектов</th>{% if request.state.is_admin %}<th style="width:1%"></th>{% endif %}</tr>
+  </thead>
+  <tbody>
+  {% for e in employees %}
+    {% if request.state.is_admin %}
+    <tr>
+      <form method="post" action="{{ url_path('/employees/' ~ e.id ~ '/update') }}">
+      <td>
+        <input name="name" value="{{ e.name }}" class="form-control form-control-sm" required>
+      </td>
+      <td>
+        <input name="tg_username" value="{{ e.tg_username or '' }}" class="form-control form-control-sm" placeholder="username">
+      </td>
+      <td>
+        <div class="input-group input-group-sm">
+          <input name="tg_chat_id" value="{{ e.tg_chat_id or '' }}" class="form-control"
+                 placeholder="например 123456789">
+          {% if e.tg_chat_id %}
+            <span class="input-group-text text-success">✓</span>
+          {% endif %}
+        </div>
+      </td>
+      <td>{{ e.projects|length }}</td>
+      <td class="text-nowrap">
+        <button class="btn btn-sm btn-primary">Сохранить</button>
+        </form>
+        <form method="post" action="{{ url_path('/employees/' ~ e.id ~ '/delete') }}" class="d-inline"
+              onsubmit="return confirm('Удалить сотрудника?');">
+          <button class="btn btn-sm btn-outline-danger">Удалить</button>
+        </form>
+      </td>
+    </tr>
+    {% else %}
+    <tr>
+      <td>{{ e.name }}</td>
+      <td>{% if e.tg_username %}@{{ e.tg_username }}{% else %}—{% endif %}</td>
+      <td>
+        {% if e.tg_chat_id %}<code>{{ e.tg_chat_id }}</code> <span class="text-success">✓</span>
+        {% else %}<span class="text-muted">не задан</span>{% endif %}
+      </td>
+      <td>{{ e.projects|length }}</td>
+    </tr>
+    {% endif %}
+  {% endfor %}
+  </tbody>
+</table>
+</div>
+
+{% endblock %}

app/templates/project_detail.html

@@ -0,0 +1,136 @@
+{% extends "base.html" %}
+{% block title %}{{ project.title }} — DLD Monitor{% endblock %}
+{% block content %}
+
+<div class="d-flex flex-column flex-md-row justify-content-between align-items-start gap-3 mb-3">
+  <div>
+    <h3 class="mb-1">{{ project.title }}</h3>
+    <div class="text-muted">
+      {% if project.deal_type.value == 'sale' %}
+        <span class="badge badge-sale">Продажа</span>
+      {% else %}
+        <span class="badge badge-rent">Аренда</span>
+      {% endif %}
+      · Владелец: {{ project.owner.name }}
+      {% if project.last_checked_at %}
+        · Проверено: {{ project.last_checked_at | msk }} МСК
+      {% endif %}
+    </div>
+    <div class="small text-muted mt-1">
+      {% if project.building %}🏢 {{ project.building }}{% endif %}
+      {% if project.bedrooms is not none %} · 🛏️ {{ project.bedrooms }} BR{% endif %}
+      {% if project.size_sqft %} · 📐 {{ "{:,.0f}".format(project.size_sqft).replace(",", " ") }} sqft{% endif %}
+      {% if project.dld_permit %} · permit: <code>{{ project.dld_permit }}</code>{% endif %}
+    </div>
+    {% if project.our_price %}
+      <div class="mt-2">Наша цена: <b>{{ "{:,.0f}".format(project.our_price).replace(",", " ") }} AED</b></div>
+    {% endif %}
+    {% if project.our_url %}
+      <div class="small mt-1">Наше объявление: <a href="{{ project.our_url }}" target="_blank" rel="noopener">{{ project.our_url }}</a></div>
+    {% endif %}
+    {% if project.notes %}<div class="mt-2 text-muted"><i>{{ project.notes }}</i></div>{% endif %}
+  </div>
+  <div class="d-flex gap-2 flex-shrink-0">
+    <form action="{{ url_path('/projects/' ~ project.id ~ '/check') }}" method="post">
+      <button class="btn btn-primary text-nowrap">Проверить сейчас</button>
+    </form>
+    {% if request.state.is_admin %}
+    <form action="{{ url_path('/projects/' ~ project.id ~ '/delete') }}" method="post"
+          onsubmit="return confirm('Удалить проект и всю историю?');">
+      <button class="btn btn-outline-danger">Удалить</button>
+    </form>
+    {% endif %}
+  </div>
+</div>
+
+{% if error %}
+  <div class="alert alert-danger">{{ error }}</div>
+{% endif %}
+{% if message %}
+  <div class="alert alert-success">{{ message }}</div>
+{% endif %}
+
+<div class="bg-white rounded shadow-sm p-3 mb-4">
+  <form method="post" action="{{ url_path('/projects/' ~ project.id ~ '/listings') }}" class="row g-2">
+    <div class="col-md-9">
+      <input name="url" type="url" required class="form-control"
+             placeholder="Вставьте URL объявления конкурента с propertyfinder.ae или bayut.com">
+    </div>
+    <div class="col-md-3">
+      <button class="btn btn-primary w-100">+ Добавить конкурента</button>
+    </div>
+  </form>
+  {% if project.our_url %}
+    <div class="mt-2">
+      <a href="{{ url_path('/projects/' ~ project.id ~ '/suggest') }}" class="btn btn-sm btn-outline-secondary">
+        🔍 Подобрать похожие на PropertyFinder
+      </a>
+      <span class="text-muted small ms-2">
+        — по зданию из вашего объявления{% if project.bedrooms is not none %}, {{ project.bedrooms }} BR{% endif %};
+        совпадения по DLD permit — первыми. Займёт ~15–20 сек.
+      </span>
+    </div>
+  {% else %}
+    <div class="mt-2 text-muted small">
+      🔍 «Подобрать похожие» появится, когда у проекта заполнены <b>наше объявление (URL)</b> и <b>спальни</b>.
+    </div>
+  {% endif %}
+</div>
+
+<h5 class="mb-3">Отслеживаемые конкуренты ({{ project.listings|length }})</h5>
+
+{% if not project.listings %}
+  <div class="alert alert-light border">
+    Пока ничего не отслеживается. Добавьте URL объявления конкурента выше.
+  </div>
+{% else %}
+  {% for l in project.listings %}
+    <div class="card mb-3 listing-card {{ 'removed' if l.status.value == 'removed' else 'active' }}">
+      <div class="card-body">
+        <div class="d-flex flex-column flex-sm-row justify-content-between gap-2">
+          <div>
+            <span class="src-{{ 'pf' if l.source.value == 'propertyfinder' else 'bayut' }}">
+              {{ 'PropertyFinder' if l.source.value == 'propertyfinder' else 'Bayut' }}
+            </span>
+            {% if l.status.value == 'removed' %}<span class="badge bg-danger ms-2">Удалено</span>{% endif %}
+            <h6 class="mt-1 mb-1">
+              <a href="{{ l.url }}" target="_blank" rel="noopener">{{ l.title or 'без названия' }}</a>
+            </h6>
+            <div class="text-muted small">
+              Брокер: {{ l.agent_name or '—' }} ({{ l.agency_name or '—' }})<br>
+              Добавлено: {{ l.first_seen_at | msk }} ·
+              Последний раз видели активным: {{ l.last_seen_at | msk }} (МСК)
+            </div>
+          </div>
+          <div class="text-sm-end flex-shrink-0">
+            <div class="fs-4 fw-bold">
+              {% if l.current_price %}
+                {{ "{:,.0f}".format(l.current_price).replace(",", " ") }} {{ l.currency or 'AED' }}
+              {% else %}—{% endif %}
+            </div>
+            {% if request.state.is_admin %}
+            <form method="post" action="{{ url_path('/listings/' ~ l.id ~ '/delete') }}"
+                  onsubmit="return confirm('Перестать отслеживать?');" class="mt-2">
+              <button class="btn btn-sm btn-outline-danger">Удалить</button>
+            </form>
+            {% endif %}
+          </div>
+        </div>
+
+        {% if l.price_history|length > 1 %}
+          <details class="mt-2">
+            <summary class="text-muted small">История цены ({{ l.price_history|length }} записей)</summary>
+            <ul class="small mt-2">
+              {% for h in l.price_history %}
+                <li>{{ h.recorded_at | msk }} —
+                    {% if h.price %}{{ "{:,.0f}".format(h.price).replace(",", " ") }} AED{% else %}—{% endif %}</li>
+              {% endfor %}
+            </ul>
+          </details>
+        {% endif %}
+      </div>
+    </div>
+  {% endfor %}
+{% endif %}
+
+{% endblock %}

app/templates/project_form.html

@@ -0,0 +1,81 @@
+{% extends "base.html" %}
+{% block title %}Новый проект — DLD Monitor{% endblock %}
+{% block content %}
+
+<h3>Новый проект</h3>
+
+{% if no_employees %}
+  <div class="alert alert-warning">
+    Сначала добавьте хотя бы одного <a href="{{ url_path('/employees') }}">сотрудника</a> — он будет получать уведомления в Telegram.
+  </div>
+{% endif %}
+
+<form method="post" action="{{ url_path('/projects/new') }}" class="bg-white p-4 rounded shadow-sm">
+  <div class="mb-3">
+    <label class="form-label">Название квартиры / лота</label>
+    <input name="title" required class="form-control" placeholder="Например: Aykon City Tower B, 2BR, Apt 1502">
+  </div>
+
+  <div class="row">
+    <div class="col-md-4 mb-3">
+      <label class="form-label">Тип сделки</label>
+      <select name="deal_type" class="form-select" required>
+        <option value="sale">Продажа</option>
+        <option value="rent">Аренда</option>
+      </select>
+    </div>
+    <div class="col-md-4 mb-3">
+      <label class="form-label">Наша цена (AED)</label>
+      <input name="our_price" type="number" step="1" class="form-control" placeholder="например 1 720 000">
+    </div>
+    <div class="col-md-4 mb-3">
+      <label class="form-label">Владелец (сотрудник)</label>
+      <select name="owner_id" class="form-select" required {% if no_employees %}disabled{% endif %}>
+        {% for e in employees %}
+          <option value="{{ e.id }}">{{ e.name }}{% if e.tg_chat_id %} ✓ TG{% endif %}</option>
+        {% endfor %}
+      </select>
+    </div>
+  </div>
+
+  <hr>
+  <p class="text-muted small mb-3">
+    Поля ниже опциональны, но позволят системе предлагать «похожие» объявления конкурентов с PF и Bayut.
+  </p>
+
+  <div class="row">
+    <div class="col-md-6 mb-3">
+      <label class="form-label">Здание / проект</label>
+      <input name="building" class="form-control" placeholder="Aykon City Tower B">
+    </div>
+    <div class="col-md-3 mb-3">
+      <label class="form-label">Спальни</label>
+      <input name="bedrooms" type="number" min="0" max="20" class="form-control" placeholder="2">
+    </div>
+    <div class="col-md-3 mb-3">
+      <label class="form-label">Площадь (sqft)</label>
+      <input name="size_sqft" type="number" step="1" class="form-control" placeholder="1058">
+    </div>
+  </div>
+
+  <div class="row">
+    <div class="col-md-8 mb-3">
+      <label class="form-label">Ссылка на наше объявление (опционально)</label>
+      <input name="our_url" type="url" class="form-control" placeholder="https://www.propertyfinder.ae/...">
+    </div>
+    <div class="col-md-4 mb-3">
+      <label class="form-label">DLD permit (опционально)</label>
+      <input name="dld_permit" class="form-control" placeholder="71-1-1-...">
+    </div>
+  </div>
+
+  <div class="mb-3">
+    <label class="form-label">Заметка</label>
+    <textarea name="notes" class="form-control" rows="2"></textarea>
+  </div>
+
+  <button class="btn btn-primary" {% if no_employees %}disabled{% endif %}>Создать</button>
+  <a href="{{ url_path('/') }}" class="btn btn-link">Отмена</a>
+</form>
+
+{% endblock %}

app/templates/projects_list.html

@@ -0,0 +1,67 @@
+{% extends "base.html" %}
+{% block title %}Проекты — DLD Monitor{% endblock %}
+{% block content %}
+
+<div class="d-flex justify-content-between align-items-center mb-3">
+  <h3 class="mb-0">Наши проекты</h3>
+  <a href="{{ url_path('/projects/new') }}" class="btn btn-primary">+ Новый проект</a>
+</div>
+
+{% if projects %}
+<div class="d-flex gap-2 mb-3">
+  <div class="stat-chip"><div class="n">{{ projects|length }}</div><div class="small text-muted">проектов</div></div>
+  <div class="stat-chip"><div class="n">{{ projects|map(attribute='listings')|map('length')|sum }}</div><div class="small text-muted">конкурентов</div></div>
+</div>
+{% endif %}
+
+{% if not projects %}
+  <div class="alert alert-light border text-center py-5">
+    Пока ни одного проекта. <a href="{{ url_path('/projects/new') }}">Добавьте первый</a>.
+  </div>
+{% else %}
+<div class="table-responsive bg-white rounded shadow-sm">
+<table class="table table-hover align-middle mb-0">
+  <thead class="table-light">
+    <tr>
+      <th>Название</th>
+      <th>Здание</th>
+      <th>Тип</th>
+      <th>Наша цена</th>
+      <th>Конкуренты</th>
+      <th>Владелец</th>
+      <th>Последняя проверка (МСК)</th>
+      <th></th>
+    </tr>
+  </thead>
+  <tbody>
+  {% for p in projects %}
+    <tr>
+      <td><a href="{{ url_path('/projects/' ~ p.id) }}">{{ p.title }}</a></td>
+      <td class="small">
+        {% if p.building %}{{ p.building }}{% else %}—{% endif %}
+        {% if p.bedrooms is not none %} · {{ p.bedrooms }}BR{% endif %}
+      </td>
+      <td>
+        {% if p.deal_type.value == 'sale' %}
+          <span class="badge badge-sale">Продажа</span>
+        {% else %}
+          <span class="badge badge-rent">Аренда</span>
+        {% endif %}
+      </td>
+      <td>{% if p.our_price %}{{ "{:,.0f}".format(p.our_price).replace(",", " ") }} AED{% else %}—{% endif %}</td>
+      <td>{{ p.listings|length }}</td>
+      <td>{{ p.owner.name if p.owner else '—' }}</td>
+      <td>{{ p.last_checked_at | msk }}</td>
+      <td>
+        <form action="{{ url_path('/projects/' ~ p.id ~ '/check') }}" method="post" class="d-inline">
+          <button class="btn btn-sm btn-outline-primary">Проверить</button>
+        </form>
+      </td>
+    </tr>
+  {% endfor %}
+  </tbody>
+</table>
+</div>
+{% endif %}
+
+{% endblock %}

app/templates/suggest.html

@@ -0,0 +1,138 @@
+{% extends "base.html" %}
+{% block title %}Похожие — {{ project.title }}{% endblock %}
+
+{% block head %}
+<style>
+  .suggest-card { cursor: pointer; transition: border-color .12s, box-shadow .12s; }
+  .suggest-card:hover { border-color: #adb5bd; }
+  .suggest-card.selected { border-color: #0d6efd; box-shadow: 0 0 0 1px #0d6efd inset; }
+  .suggest-card a { cursor: pointer; }
+  /* Sticky action bar so the submit button is always reachable, esp. on mobile */
+  .bulk-bar {
+    position: sticky; bottom: 0; z-index: 1020;
+    background: #fff; border-top: 1px solid #e9ecef;
+    padding: .6rem .25rem; margin-top: 1rem;
+    box-shadow: 0 -4px 12px rgba(0,0,0,.05);
+  }
+  @media (max-width: 575.98px) {
+    .bulk-bar #bulk-submit { flex: 1; }
+  }
+</style>
+{% endblock %}
+
+{% block content %}
+
+<div class="mb-3">
+  <a href="{{ url_path('/projects/' ~ project.id) }}" class="btn btn-sm btn-link px-0">← к проекту</a>
+</div>
+
+<h3 class="mb-1">Подсказки похожих объявлений</h3>
+<p class="text-muted">
+  Ищем объявления в том же здании{% if project.bedrooms is not none %}, {{ project.bedrooms }} спален{% endif %},
+  тип сделки: {{ project.deal_type.value }}.
+  {% if our_permit %}Совпадение по DLD permit (<code>{{ our_permit }}</code>) — это тот же объект, такие показаны первыми. {% endif %}
+  Отметьте подходящие объявления галочкой (можно несколько) и нажмите «Отслеживать выбранные» — они попадут в трекинг.
+</p>
+
+<form id="bulk-form" method="post" action="{{ url_path('/projects/' ~ project.id ~ '/listings/bulk') }}">
+{% for src_key, src_label in [('propertyfinder', 'PropertyFinder'), ('bayut', 'Bayut')] %}
+  <h5 class="mt-4">
+    <span class="src-{{ 'pf' if src_key == 'propertyfinder' else 'bayut' }}">{{ src_label }}</span>
+    {% if src_key == 'bayut' and not bayut_enabled %}
+      <small class="text-muted">— ⏸ временно отключён</small>
+    {% else %}
+      <small class="text-muted">— найдено {{ suggestions[src_key]|length }}</small>
+    {% endif %}
+  </h5>
+  {% if src_key == 'bayut' and not bayut_enabled %}
+    <div class="text-muted small">Bayut перешёл на защищённый рендеринг — поиск и трекинг временно недоступны.</div>
+  {% elif not suggestions[src_key] %}
+    <div class="text-muted small">Ничего не нашли (или площадка заблокировала запрос).</div>
+  {% else %}
+    {% for s in suggestions[src_key] %}
+      {% set same_permit = our_permit and s.permit_number and s.permit_number == our_permit %}
+      <label class="card mb-2 suggest-card{% if same_permit %} border-success{% endif %}">
+        <div class="card-body py-2">
+          <div class="d-flex gap-2 align-items-start">
+            <input class="form-check-input suggest-check flex-shrink-0 mt-1" type="checkbox"
+                   name="urls" value="{{ s.url }}">
+            <div class="flex-grow-1">
+              <div class="d-flex flex-column flex-sm-row justify-content-between gap-1">
+                <div>
+                  {% if same_permit %}<span class="badge bg-success">🎯 тот же permit</span> {% endif %}
+                  <a href="{{ s.url }}" target="_blank" rel="noopener">{{ s.title or 'без названия' }}</a>
+                  <div class="small text-muted">
+                    {{ s.agent_name or '—' }} ({{ s.agency_name or '—' }})
+                    {% if s.permit_number %} · permit <code>{{ s.permit_number }}</code>{% endif %}
+                  </div>
+                </div>
+                <div class="fw-bold text-sm-end text-nowrap">
+                  {% if s.price %}{{ "{:,.0f}".format(s.price).replace(",", " ") }} {{ s.currency or 'AED' }}{% else %}—{% endif %}
+                </div>
+              </div>
+            </div>
+          </div>
+        </div>
+      </label>
+    {% endfor %}
+  {% endif %}
+{% endfor %}
+
+  <div class="bulk-bar">
+    <div class="d-flex align-items-center gap-3">
+      <div class="form-check m-0">
+        <input class="form-check-input" type="checkbox" id="select-all">
+        <label class="form-check-label small" for="select-all">Выбрать все</label>
+      </div>
+      <button type="submit" class="btn btn-primary ms-auto" id="bulk-submit" disabled>
+        + Отслеживать выбранные (<span id="sel-count">0</span>)
+      </button>
+    </div>
+  </div>
+</form>
+
+{% endblock %}
+
+{% block scripts %}
+<script>
+(function () {
+  const form = document.getElementById('bulk-form');
+  if (!form) return;
+  const checks = Array.from(form.querySelectorAll('.suggest-check'));
+  const selectAll = document.getElementById('select-all');
+  const countEl = document.getElementById('sel-count');
+  const submitBtn = document.getElementById('bulk-submit');
+
+  function refresh() {
+    let n = 0;
+    checks.forEach(c => {
+      const card = c.closest('.suggest-card');
+      if (card) card.classList.toggle('selected', c.checked);
+      if (c.checked) n++;
+    });
+    countEl.textContent = n;
+    submitBtn.disabled = n === 0;
+    if (selectAll) selectAll.checked = n > 0 && n === checks.length;
+  }
+
+  checks.forEach(c => c.addEventListener('change', refresh));
+
+  if (selectAll) {
+    selectAll.addEventListener('change', () => {
+      checks.forEach(c => { c.checked = selectAll.checked; });
+      refresh();
+    });
+  }
+
+  // Confirm before submitting a large batch — each add re-fetches the page.
+  form.addEventListener('submit', e => {
+    const n = checks.filter(c => c.checked).length;
+    if (n === 0) { e.preventDefault(); return; }
+    submitBtn.disabled = true;
+    submitBtn.textContent = 'Добавляем…';
+  });
+
+  refresh();
+})();
+</script>
+{% endblock %}

app/web.py

@@ -0,0 +1,386 @@
+"""FastAPI web app — UI for managing projects, competitor listings, employees."""
+
+from __future__ import annotations
+
+from datetime import timedelta
+from pathlib import Path
+
+from fastapi import Depends, FastAPI, Form, HTTPException, Request
+from fastapi.responses import HTMLResponse, RedirectResponse
+from urllib.parse import quote
+from fastapi.templating import Jinja2Templates
+from sqlalchemy.orm import Session, joinedload
+
+from app.auth import (
+    ADMIN_COOKIE,
+    COOKIE_MAX_AGE,
+    admin_configured,
+    admin_token,
+    pin_ok,
+    request_is_admin,
+)
+from app.config import BASE_DIR, settings
+from app.db import get_db, init_db
+from app.models import CompetitorListing, DealType, Employee, Project
+from app.services.monitor import (
+    BAYUT_ENABLED,
+    add_competitor_url,
+    add_competitor_urls,
+    resolve_our_permit,
+    run_check_for_project,
+    suggest_similar,
+)
+
+app = FastAPI(title="DLD Monitor")
+templates = Jinja2Templates(directory=str(Path(BASE_DIR) / "app" / "templates"))
+
+# Timestamps are stored as naive UTC (datetime.utcnow). Show them in Moscow time
+# (UTC+3) in the UI via the `msk` Jinja filter.
+_MSK_OFFSET = timedelta(hours=3)
+
+
+def _to_msk(dt, fmt: str = "%Y-%m-%d %H:%M"):
+    if dt is None:
+        return "—"
+    return (dt + _MSK_OFFSET).strftime(fmt)
+
+
+templates.env.filters["msk"] = _to_msk
+
+
+def web_path(path: str = "/") -> str:
+    base = settings.public_base_path.rstrip("/")
+    if not path.startswith("/"):
+        path = "/" + path
+    return f"{base}{path}" if base else path
+
+
+templates.env.globals["url_path"] = web_path
+
+
+@app.on_event("startup")
+def _startup() -> None:
+    init_db()
+
+
+@app.middleware("http")
+async def _admin_state(request: Request, call_next):
+    """Expose admin status to every route and template (request.state.is_admin)."""
+    request.state.is_admin = request_is_admin(request)
+    request.state.admin_configured = admin_configured()
+    return await call_next(request)
+
+
+@app.get("/healthz")
+def healthz() -> dict[str, str]:
+    return {"status": "ok"}
+
+
+def admin_required(request: Request) -> None:
+    """Dependency: block the request unless the session is in admin mode."""
+    if not request.state.is_admin:
+        raise HTTPException(403, "Действие доступно только администратору. Войдите в режим админа.")
+
+
+def _safe_next(next_url: str) -> str:
+    """Only allow same-site relative redirects after login."""
+    return next_url if next_url.startswith("/") and not next_url.startswith("//") else "/"
+
+
+# --- Admin session --------------------------------------------------------
+
+@app.get("/admin/login", response_class=HTMLResponse)
+def admin_login_page(request: Request, next: str = "/", error: str | None = None):
+    return templates.TemplateResponse(
+        "admin_login.html",
+        {"request": request, "next": _safe_next(next), "error": error, "flash": None},
+    )
+
+
+@app.post("/admin/login")
+def admin_login(request: Request, pin: str = Form(...), next: str = Form("/")):
+    dest = _safe_next(next)
+    if not pin_ok(pin):
+        return templates.TemplateResponse(
+            "admin_login.html",
+            {"request": request, "next": dest, "error": "Неверный PIN", "flash": None},
+            status_code=401,
+        )
+    resp = RedirectResponse(web_path(dest), status_code=303)
+    resp.set_cookie(
+        ADMIN_COOKIE, admin_token(), max_age=COOKIE_MAX_AGE,
+        httponly=True, samesite="lax",
+    )
+    return resp
+
+
+@app.post("/admin/logout")
+def admin_logout(next: str = Form("/")):
+    resp = RedirectResponse(web_path(_safe_next(next)), status_code=303)
+    resp.delete_cookie(ADMIN_COOKIE)
+    return resp
+
+
+def _opt_float(s: str) -> float | None:
+    s = (s or "").strip()
+    if not s:
+        return None
+    try:
+        return float(s)
+    except ValueError:
+        return None
+
+
+def _opt_int(s: str) -> int | None:
+    s = (s or "").strip()
+    if not s:
+        return None
+    try:
+        return int(s)
+    except ValueError:
+        return None
+
+
+# --- Projects -------------------------------------------------------------
+
+@app.get("/", response_class=HTMLResponse)
+def projects_list(request: Request, db: Session = Depends(get_db)):
+    projects = (
+        db.query(Project)
+        .options(joinedload(Project.owner), joinedload(Project.listings))
+        .order_by(Project.created_at.desc())
+        .all()
+    )
+    return templates.TemplateResponse(
+        "projects_list.html", {"request": request, "projects": projects, "flash": None}
+    )
+
+
+@app.get("/projects/new", response_class=HTMLResponse)
+def new_project_form(request: Request, db: Session = Depends(get_db)):
+    employees = db.query(Employee).order_by(Employee.name).all()
+    return templates.TemplateResponse(
+        "project_form.html",
+        {
+            "request": request,
+            "employees": employees,
+            "no_employees": not employees,
+            "flash": None,
+        },
+    )
+
+
+@app.post("/projects/new")
+def create_project(
+    title: str = Form(...),
+    deal_type: str = Form(...),
+    owner_id: int = Form(...),
+    our_price: str = Form(""),
+    building: str = Form(""),
+    bedrooms: str = Form(""),
+    size_sqft: str = Form(""),
+    our_url: str = Form(""),
+    dld_permit: str = Form(""),
+    notes: str = Form(""),
+    db: Session = Depends(get_db),
+):
+    owner = db.get(Employee, owner_id)
+    if not owner:
+        raise HTTPException(404, "Employee not found")
+    project = Project(
+        title=title.strip(),
+        deal_type=DealType(deal_type),
+        our_price=_opt_float(our_price),
+        building=building.strip() or None,
+        bedrooms=_opt_int(bedrooms),
+        size_sqft=_opt_float(size_sqft),
+        our_url=our_url.strip() or None,
+        dld_permit=dld_permit.strip() or None,
+        notes=notes.strip() or None,
+        owner_id=owner.id,
+    )
+    db.add(project)
+    db.commit()
+    return RedirectResponse(web_path(f"/projects/{project.id}"), status_code=303)
+
+
+@app.get("/projects/{project_id}", response_class=HTMLResponse)
+def project_detail(
+    project_id: int,
+    request: Request,
+    error: str | None = None,
+    message: str | None = None,
+    db: Session = Depends(get_db),
+):
+    project = (
+        db.query(Project)
+        .options(joinedload(Project.owner), joinedload(Project.listings))
+        .filter(Project.id == project_id)
+        .first()
+    )
+    if not project:
+        raise HTTPException(404, "Project not found")
+    return templates.TemplateResponse(
+        "project_detail.html",
+        {"request": request, "project": project, "error": error, "message": message, "flash": None},
+    )
+
+
+@app.post("/projects/{project_id}/check")
+def project_check_now(project_id: int, db: Session = Depends(get_db)):
+    if not db.get(Project, project_id):
+        raise HTTPException(404, "Project not found")
+    db.close()
+    run_check_for_project(project_id)
+    return RedirectResponse(web_path(f"/projects/{project_id}"), status_code=303)
+
+
+@app.post("/projects/{project_id}/delete")
+def project_delete(project_id: int, db: Session = Depends(get_db), _: None = Depends(admin_required)):
+    project = db.get(Project, project_id)
+    if not project:
+        raise HTTPException(404, "Project not found")
+    db.delete(project)
+    db.commit()
+    return RedirectResponse(web_path("/"), status_code=303)
+
+
+# --- Competitor listings (per project) ------------------------------------
+
+@app.post("/projects/{project_id}/listings")
+def add_listing(project_id: int, url: str = Form(...), db: Session = Depends(get_db)):
+    project = db.get(Project, project_id)
+    if not project:
+        raise HTTPException(404, "Project not found")
+    listing, err = add_competitor_url(db, project, url)
+    if err:
+        return RedirectResponse(web_path(f"/projects/{project_id}?error={quote(err)}"), status_code=303)
+    return RedirectResponse(
+        web_path(f"/projects/{project_id}?message=Добавлено"), status_code=303,
+    )
+
+
+@app.post("/projects/{project_id}/listings/bulk")
+def add_listings_bulk(
+    project_id: int,
+    urls: list[str] = Form(default=[]),
+    db: Session = Depends(get_db),
+):
+    project = db.get(Project, project_id)
+    if not project:
+        raise HTTPException(404, "Project not found")
+    if not urls:
+        return RedirectResponse(
+            web_path(f"/projects/{project_id}?error={quote('Ничего не выбрано')}"),
+            status_code=303,
+        )
+    result = add_competitor_urls(db, project, urls)
+    parts = [f"Добавлено: {result['added']}"]
+    if result["skipped"]:
+        parts.append(f"уже были: {result['skipped']}")
+    if result["errors"]:
+        parts.append(f"ошибок: {len(result['errors'])}")
+    msg = " · ".join(parts)
+    return RedirectResponse(
+        web_path(f"/projects/{project_id}?message={quote(msg)}"), status_code=303,
+    )
+
+
+@app.post("/listings/{listing_id}/delete")
+def listing_delete(listing_id: int, db: Session = Depends(get_db), _: None = Depends(admin_required)):
+    listing = db.get(CompetitorListing, listing_id)
+    if not listing:
+        raise HTTPException(404, "Listing not found")
+    project_id = listing.project_id
+    db.delete(listing)
+    db.commit()
+    return RedirectResponse(web_path(f"/projects/{project_id}"), status_code=303)
+
+
+@app.get("/projects/{project_id}/suggest", response_class=HTMLResponse)
+def project_suggest(project_id: int, request: Request, db: Session = Depends(get_db)):
+    project = (
+        db.query(Project)
+        .options(joinedload(Project.listings))
+        .filter(Project.id == project_id)
+        .first()
+    )
+    if not project:
+        raise HTTPException(404, "Project not found")
+    our_permit = resolve_our_permit(project)
+    suggestions = suggest_similar(project, our_permit=our_permit)
+    return templates.TemplateResponse(
+        "suggest.html",
+        {
+            "request": request,
+            "project": project,
+            "suggestions": suggestions,
+            "our_permit": our_permit,
+            "bayut_enabled": BAYUT_ENABLED,
+            "flash": None,
+        },
+    )
+
+
+# --- Employees ------------------------------------------------------------
+
+@app.get("/employees", response_class=HTMLResponse)
+def employees_page(request: Request, db: Session = Depends(get_db)):
+    employees = (
+        db.query(Employee).options(joinedload(Employee.projects)).order_by(Employee.name).all()
+    )
+    return templates.TemplateResponse(
+        "employees.html", {"request": request, "employees": employees, "flash": None}
+    )
+
+
+@app.post("/employees")
+def employee_create(
+    name: str = Form(...),
+    tg_username: str = Form(""),
+    db: Session = Depends(get_db),
+):
+    e = Employee(name=name.strip(), tg_username=tg_username.strip().lstrip("@") or None)
+    db.add(e)
+    db.commit()
+    return RedirectResponse(web_path("/employees"), status_code=303)
+
+
+@app.post("/employees/{employee_id}/update")
+def employee_update(
+    employee_id: int,
+    name: str = Form(...),
+    tg_username: str = Form(""),
+    tg_chat_id: str = Form(""),
+    db: Session = Depends(get_db),
+    _: None = Depends(admin_required),
+):
+    e = db.get(Employee, employee_id)
+    if not e:
+        raise HTTPException(404, "Employee not found")
+    e.name = name.strip()
+    e.tg_username = tg_username.strip().lstrip("@") or None
+    new_chat_id = tg_chat_id.strip() or None
+    if new_chat_id and new_chat_id != e.tg_chat_id:
+        clash = (
+            db.query(Employee)
+            .filter(Employee.tg_chat_id == new_chat_id, Employee.id != e.id)
+            .first()
+        )
+        if clash:
+            raise HTTPException(400, f"chat_id {new_chat_id} уже привязан к сотруднику «{clash.name}»")
+    e.tg_chat_id = new_chat_id
+    db.commit()
+    return RedirectResponse(web_path("/employees"), status_code=303)
+
+
+@app.post("/employees/{employee_id}/delete")
+def employee_delete(employee_id: int, db: Session = Depends(get_db), _: None = Depends(admin_required)):
+    e = db.get(Employee, employee_id)
+    if not e:
+        raise HTTPException(404, "Employee not found")
+    if e.projects:
+        raise HTTPException(400, "Сотрудник связан с проектами — сначала переназначьте или удалите проекты")
+    db.delete(e)
+    db.commit()
+    return RedirectResponse(web_path("/employees"), status_code=303)

diagnose.py

@@ -0,0 +1,89 @@
+r"""Одноразовая диагностика: проверяет всю цепочку мониторинга.
+Запуск: & "<...>\.venv\Scripts\python.exe" diagnose.py
+Только чтение БД + проверка токена TG + живой re-fetch. Сообщения НЕ шлёт."""
+from __future__ import annotations
+
+import sys
+
+import httpx
+
+from app.config import settings
+from app.db import SessionLocal
+from app.models import CompetitorListing, Employee, PriceHistory, Project
+from app.services.monitor import check_project
+
+
+def line(c: str = "-") -> None:
+    print(c * 60)
+
+
+def main() -> None:
+    db = SessionLocal()
+    try:
+        # --- 1. Содержимое БД ---
+        line("=")
+        print("1) ДАННЫЕ В БАЗЕ")
+        line("=")
+        projects = db.query(Project).all()
+        if not projects:
+            print("  ! В базе нет ни одного проекта.")
+            return
+        for p in projects:
+            owner = p.owner
+            chat = owner.tg_chat_id if owner else None
+            chat_str = chat if chat else "НЕ ЗАДАН (уведомления не дойдут!)"
+            print(f"\n  Проект #{p.id}: {p.title}")
+            print(f"    тип: {p.deal_type.value} | наша цена: {p.our_price}")
+            print(f"    владелец: {owner.name if owner else '—'} | tg_chat_id: {chat_str}")
+            print(f"    последняя проверка: {p.last_checked_at or 'ещё не было'}")
+            listings = p.listings
+            print(f"    отслеживаемых объявлений конкурентов: {len(listings)}")
+            for l in listings:
+                hist = db.query(PriceHistory).filter(PriceHistory.listing_id == l.id).count()
+                price = f"{l.current_price:,.0f} {l.currency}".replace(",", " ") if l.current_price else "БЕЗ ЦЕНЫ"
+                print(f"      - [{l.source.value}] {l.status.value} | {price} | история: {hist} зап.")
+                print(f"        {(l.title or 'без названия')[:70]}")
+                print(f"        брокер: {l.agency_name or '—'} | {l.url}")
+
+        # --- 2. Telegram токен ---
+        line("=")
+        print("2) TELEGRAM")
+        line("=")
+        if not settings.tg_bot_token:
+            print("  ! TG_BOT_TOKEN не задан в .env")
+        else:
+            try:
+                r = httpx.get(
+                    f"https://api.telegram.org/bot{settings.tg_bot_token}/getMe",
+                    timeout=15.0,
+                )
+                if r.status_code == 200 and r.json().get("ok"):
+                    bot = r.json()["result"]
+                    print(f"  OK токен валиден: @{bot.get('username')} ({bot.get('first_name')})")
+                else:
+                    print(f"  ! getMe вернул {r.status_code}: {r.text}")
+            except Exception as e:
+                print(f"  ! Ошибка связи с Telegram: {e}")
+        print(f"  ADMIN_CHAT_ID: {settings.admin_chat_id or 'не задан'}")
+
+        # --- 3. Живой re-fetch (то, что делает сканер раз в 4 ч) ---
+        line("=")
+        print("3) ЖИВАЯ ПЕРЕПРОВЕРКА (re-fetch всех объявлений)")
+        line("=")
+        for p in projects:
+            print(f"\n  Проект #{p.id}: {p.title} — проверяю {len(p.listings)} объявл...")
+            changes = check_project(db, p)
+            if changes:
+                print(f"    обнаружено изменений: {len(changes)} (в реальном прогоне ушли бы в TG):")
+                for c in changes:
+                    print("      " + c.replace("\n", " | ").replace("<b>", "").replace("</b>", ""))
+            else:
+                print("    изменений нет — все объявления успешно перезагружены, цены прежние.")
+        line("=")
+        print("Готово. Если у объявлений есть цены и статус active, а токен валиден — цепочка рабочая.")
+    finally:
+        db.close()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

docker-compose.yml

@@ -0,0 +1,37 @@
+services:
+  web:
+    build: .
+    image: dld-permit-monitor:latest
+    container_name: dld-web
+    restart: unless-stopped
+    env_file: .env
+    environment:
+      WEB_HOST: 0.0.0.0
+      WEB_PORT: 8000
+    ports:
+      - "8080:8000"
+    volumes:
+      - ./data:/app/data
+    command: ["python", "run_web.py"]
+
+  bot:
+    image: dld-permit-monitor:latest
+    depends_on:
+      - web
+    container_name: dld-bot
+    restart: unless-stopped
+    env_file: .env
+    volumes:
+      - ./data:/app/data
+    command: ["python", "-m", "app.bot"]
+
+  scheduler:
+    image: dld-permit-monitor:latest
+    depends_on:
+      - web
+    container_name: dld-scheduler
+    restart: unless-stopped
+    env_file: .env
+    volumes:
+      - ./data:/app/data
+    command: ["python", "-m", "app.scheduler"]

k8s/configmap.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: monitoring-pf-config
+  namespace: monitoring-pf
+data:
+  WEB_HOST: "0.0.0.0"
+  WEB_PORT: "8000"
+  PUBLIC_BASE_PATH: "/api/monitoring-pf"
+  DATABASE_URL: "sqlite:////data/monitor.db"
+  SCRAPE_INTERVAL_HOURS: "4"

k8s/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: monitoring-pf
+
+resources:
+  - namespace.yaml
+  - configmap.yaml
+  - secrets.yaml
+  - server-deployment.yaml
+  - server-service.yaml

k8s/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring-pf

k8s/secrets.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: monitoring-pf-secrets
+  namespace: monitoring-pf
+type: Opaque
+stringData:
+  TG_BOT_TOKEN: "CHANGE_ME"
+  ADMIN_CHAT_ID: ""
+  ADMIN_PIN: "CHANGE_ME"

k8s/server-deployment.yaml

@@ -0,0 +1,107 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: monitoring-pf-data
+  namespace: monitoring-pf
+spec:
+  accessModes: ["ReadWriteOnce"]
+  storageClassName: local-path
+  resources:
+    requests:
+      storage: 5Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: monitoring-pf-server
+  namespace: monitoring-pf
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: monitoring-pf-server
+  template:
+    metadata:
+      labels:
+        app: monitoring-pf-server
+    spec:
+      terminationGracePeriodSeconds: 20
+      securityContext:
+        fsGroup: 1000
+      containers:
+        - name: web
+          image: localhost:30300/admin/monitoring-pf-server:latest
+          command: ["python", "run_web.py"]
+          ports:
+            - containerPort: 8000
+          envFrom:
+            - configMapRef:
+                name: monitoring-pf-config
+            - secretRef:
+                name: monitoring-pf-secrets
+          volumeMounts:
+            - name: app-data
+              mountPath: /data
+          startupProbe:
+            httpGet:
+              path: /healthz
+              port: 8000
+            periodSeconds: 5
+            failureThreshold: 30
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 8000
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: 8000
+            periodSeconds: 5
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: 800m
+              memory: 768Mi
+        - name: bot
+          image: localhost:30300/admin/monitoring-pf-server:latest
+          command: ["python", "-m", "app.bot"]
+          envFrom:
+            - configMapRef:
+                name: monitoring-pf-config
+            - secretRef:
+                name: monitoring-pf-secrets
+          volumeMounts:
+            - name: app-data
+              mountPath: /data
+          resources:
+            requests:
+              cpu: 25m
+              memory: 96Mi
+            limits:
+              cpu: 300m
+              memory: 384Mi
+        - name: scheduler
+          image: localhost:30300/admin/monitoring-pf-server:latest
+          command: ["python", "-m", "app.scheduler"]
+          envFrom:
+            - configMapRef:
+                name: monitoring-pf-config
+            - secretRef:
+                name: monitoring-pf-secrets
+          volumeMounts:
+            - name: app-data
+              mountPath: /data
+          resources:
+            requests:
+              cpu: 50m
+              memory: 128Mi
+            limits:
+              cpu: 600m
+              memory: 768Mi
+      volumes:
+        - name: app-data
+          persistentVolumeClaim:
+            claimName: monitoring-pf-data

k8s/server-service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: monitoring-pf-server
+  namespace: monitoring-pf
+  annotations:
+    portal.estateliga.work/enabled: "true"
+    portal.estateliga.work/name: "Мониторинг PF"
+    portal.estateliga.work/description: "Мониторинг объявлений PropertyFinder/Bayut по DLD Permit"
+    portal.estateliga.work/icon: "pulse"
+    portal.estateliga.work/path: "/api/monitoring-pf"
+    portal.estateliga.work/code: "monitoring_pf"
+spec:
+  selector:
+    app: monitoring-pf-server
+  ports:
+    - port: 80
+      targetPort: 8000

requirements.txt

@@ -0,0 +1,14 @@
+fastapi==0.115.6
+uvicorn[standard]==0.32.1
+jinja2==3.1.4
+python-multipart==0.0.20
+sqlalchemy==2.0.36
+httpx==0.28.1
+brotli==1.2.0  # lets httpx decode Content-Encoding: br (Bayut serves Brotli)
+beautifulsoup4==4.12.3
+lxml==5.3.0
+apscheduler==3.11.0
+python-telegram-bot==21.9
+python-dotenv==1.0.1
+pydantic==2.10.4
+pydantic-settings==2.7.0

run_bot.bat

@@ -0,0 +1,4 @@
+@echo off
+cd /d "%~dp0"
+".venv\Scripts\python.exe" -m app.bot
+pause

run_scheduler.bat

@@ -0,0 +1,4 @@
+@echo off
+cd /d "%~dp0"
+".venv\Scripts\python.exe" -m app.scheduler
+pause

run_web.bat

@@ -0,0 +1,4 @@
+@echo off
+cd /d "%~dp0"
+".venv\Scripts\python.exe" run_web.py
+pause

run_web.py

@@ -0,0 +1,13 @@
+"""Convenience launcher for the web UI: `python run_web.py`."""
+
+import uvicorn
+
+from app.config import settings
+
+if __name__ == "__main__":
+    uvicorn.run(
+        "app.web:app",
+        host=settings.web_host,
+        port=settings.web_port,
+        reload=False,
+    )