feat: parse project metadata from PF links

.gitea/scripts/hygiene-check.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+fail=0
+
+while IFS= read -r -d '' path; do
+  base="$(basename "$path")"
+  case "$base" in
+    .DS_Store|.env)
+      echo "::error file=$path::tracked local-only file is forbidden"
+      fail=1
+      ;;
+  esac
+
+  case "$path" in
+    *node_modules/*|node_modules/*)
+      echo "::error file=$path::tracked node_modules content is forbidden"
+      fail=1
+      ;;
+    *.tmp|*.temp|*.bak|*.orig|*.rej|*.zip|*.tar|*.tar.gz|*.tgz|*.rar|*.7z)
+      echo "::error file=$path::tracked temporary/archive artifact is forbidden"
+      fail=1
+      ;;
+  esac
+
+  if [ -f "$path" ]; then
+    size="$(wc -c < "$path" | tr -d ' ')"
+    if [ "${size:-0}" -gt 52428800 ]; then
+      echo "::error file=$path::tracked file is larger than 50 MiB"
+      fail=1
+    fi
+  fi
+done < <(git ls-files -z)
+
+exit "$fail"

.gitea/workflows/ci.yml

@@ -0,0 +1,41 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  hygiene:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: bash .gitea/scripts/hygiene-check.sh
+
+  go:
+    runs-on: ubuntu-latest
+    needs: hygiene
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+      - run: go build ./...
+      - run: go test ./...
+      - uses: golangci/golangci-lint-action@v7
+        with:
+          version: v2.4
+          args: --config .golangci.yml ./...
+
+  python:
+    runs-on: ubuntu-latest
+    needs: hygiene
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          if ! python3 -m pip --version; then
+            python3 -m ensurepip --upgrade || (apt-get update && apt-get install -y python3-pip)
+          fi
+      - run: python3 -m pip install -r requirements.txt
+      - run: python3 -m compileall app
+      - run: python3 -m unittest discover -s tests

.golangci.yml

@@ -0,0 +1,36 @@
+version: "2"
+
+run:
+  timeout: 3m
+
+linters:
+  default: none
+  enable:
+    - errcheck
+    - govet
+    - ineffassign
+    - staticcheck
+    - unused
+  settings:
+    errcheck:
+      check-type-assertions: true
+      check-blank: false
+      exclude-functions:
+        - (io.Closer).Close
+        - (net/http.ResponseWriter).Write
+        - (*encoding/json.Encoder).Encode
+        - io.Copy
+        - fmt.Fprintf
+        - (github.com/jackc/pgx/v5.Tx).Rollback
+        - os.RemoveAll
+    staticcheck:
+      checks: ["all", "-SA1019", "-ST1000", "-ST1005", "-ST1020", "-ST1021", "-ST1022"]
+  exclusions:
+    rules:
+      - path: _test\.go
+        linters:
+          - errcheck
+
+issues:
+  max-issues-per-linter: 0
+  max-same-issues: 0

app/db.py

@@ -56,3 +56,7 @@ def _migrate_competitor_listings_auto_fields() -> None:
             conn.execute(text("ALTER TABLE competitor_listings ADD COLUMN permit_number VARCHAR(100)"))
         if "auto_discovered" not in columns:
             conn.execute(text("ALTER TABLE competitor_listings ADD COLUMN auto_discovered BOOLEAN NOT NULL DEFAULT 0"))
+        if "permit_missing_checks" not in columns:
+            conn.execute(
+                text("ALTER TABLE competitor_listings ADD COLUMN permit_missing_checks INTEGER NOT NULL DEFAULT 0")
+            )

app/models.py

@@ -1,5 +1,8 @@
+from __future__ import annotations
+
 from datetime import datetime
 from enum import Enum
+from typing import Optional
 
 from sqlalchemy import Boolean, DateTime, Enum as SAEnum, Float, ForeignKey, Integer, String, Text, UniqueConstraint
 from sqlalchemy.orm import Mapped, mapped_column, relationship
@@ -27,9 +30,9 @@ class Employee(Base):
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
     name: Mapped[str] = mapped_column(String(200))
-    portal_user_id: Mapped[str | None] = mapped_column(String(100), unique=True, index=True, nullable=True)
-    tg_chat_id: Mapped[str | None] = mapped_column(String(64), unique=True, nullable=True)
-    tg_username: Mapped[str | None] = mapped_column(String(200), nullable=True)
+    portal_user_id: Mapped[Optional[str]] = mapped_column(String(100), unique=True, index=True, nullable=True)
+    tg_chat_id: Mapped[Optional[str]] = mapped_column(String(64), unique=True, nullable=True)
+    tg_username: Mapped[Optional[str]] = mapped_column(String(200), nullable=True)
     created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
 
     projects: Mapped[list["Project"]] = relationship(back_populates="owner")
@@ -43,21 +46,21 @@ class Project(Base):
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
     title: Mapped[str] = mapped_column(String(300))
     deal_type: Mapped[DealType] = mapped_column(SAEnum(DealType))
-    our_price: Mapped[float | None] = mapped_column(Float, nullable=True)
-    notes: Mapped[str | None] = mapped_column(Text, nullable=True)
+    our_price: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+    notes: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
 
     # Опциональные параметры — используются для подсказок похожих объявлений
-    dld_permit: Mapped[str | None] = mapped_column(String(100), index=True, nullable=True)
-    building: Mapped[str | None] = mapped_column(String(300), nullable=True)
-    bedrooms: Mapped[int | None] = mapped_column(Integer, nullable=True)
-    size_sqft: Mapped[float | None] = mapped_column(Float, nullable=True)
-    our_url: Mapped[str | None] = mapped_column(Text, nullable=True)
+    dld_permit: Mapped[Optional[str]] = mapped_column(String(100), index=True, nullable=True)
+    building: Mapped[Optional[str]] = mapped_column(String(300), nullable=True)
+    bedrooms: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
+    size_sqft: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+    our_url: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
 
     owner_id: Mapped[int] = mapped_column(ForeignKey("employees.id"))
     owner: Mapped[Employee] = relationship(back_populates="projects")
 
     created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
-    last_checked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+    last_checked_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
     listings: Mapped[list["CompetitorListing"]] = relationship(
         back_populates="project", cascade="all, delete-orphan"
@@ -77,14 +80,15 @@ class CompetitorListing(Base):
     source: Mapped[Source] = mapped_column(SAEnum(Source))
     external_id: Mapped[str] = mapped_column(String(100))  # ID на стороне PF/Bayut
     url: Mapped[str] = mapped_column(Text)
-    title: Mapped[str | None] = mapped_column(String(500), nullable=True)
-    agent_name: Mapped[str | None] = mapped_column(String(300), nullable=True)
-    agency_name: Mapped[str | None] = mapped_column(String(300), nullable=True)
-    permit_number: Mapped[str | None] = mapped_column(String(100), nullable=True)
+    title: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
+    agent_name: Mapped[Optional[str]] = mapped_column(String(300), nullable=True)
+    agency_name: Mapped[Optional[str]] = mapped_column(String(300), nullable=True)
+    permit_number: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)
     auto_discovered: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    permit_missing_checks: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
 
-    current_price: Mapped[float | None] = mapped_column(Float, nullable=True)
-    currency: Mapped[str | None] = mapped_column(String(10), nullable=True, default="AED")
+    current_price: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+    currency: Mapped[Optional[str]] = mapped_column(String(10), nullable=True, default="AED")
     status: Mapped[ListingStatus] = mapped_column(SAEnum(ListingStatus), default=ListingStatus.ACTIVE)
 
     first_seen_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
@@ -102,5 +106,5 @@ class PriceHistory(Base):
     listing_id: Mapped[int] = mapped_column(ForeignKey("competitor_listings.id"))
     listing: Mapped[CompetitorListing] = relationship(back_populates="price_history")
 
-    price: Mapped[float | None] = mapped_column(Float, nullable=True)
+    price: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
     recorded_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)

app/scrapers/base.py

@@ -35,6 +35,9 @@ class ScrapedListing:
     permit_number: str | None
     agent_name: str | None
     agency_name: str | None
+    building: str | None = None
+    bedrooms: int | None = None
+    size_sqft: float | None = None
     is_active: bool = True
 
 

app/scrapers/propertyfinder.py

@@ -100,6 +100,110 @@ def _extract_permit(item: dict) -> str | None:
     return None
 
 
+def _parse_int(value) -> int | None:
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, (int, float)):
+        return int(value)
+    text = str(value).strip().lower()
+    if text in {"studio", "студия"}:
+        return 0
+    m = re.search(r"\d+", text)
+    return int(m.group(0)) if m else None
+
+
+def _extract_bedrooms(item: dict) -> int | None:
+    for key in ("bedrooms", "bedroom", "beds", "rooms", "bedroom_count", "bedrooms_count"):
+        value = item.get(key)
+        if isinstance(value, dict):
+            value = value.get("value") or value.get("count") or value.get("name")
+        parsed = _parse_int(value)
+        if parsed is not None:
+            return parsed
+
+    for node in _walk(item):
+        if not isinstance(node, dict):
+            continue
+        name = str(node.get("name") or node.get("label") or node.get("key") or "").lower()
+        if "bed" not in name and "спал" not in name:
+            continue
+        parsed = _parse_int(node.get("value") or node.get("count") or node.get("text"))
+        if parsed is not None:
+            return parsed
+    return None
+
+
+def _area_to_sqft(value, unit: str | None = None) -> float | None:
+    parsed = parse_price(value)
+    if parsed is None:
+        return None
+    unit_text = (unit or "").lower()
+    if any(token in unit_text for token in ("sqm", "sq m", "m2", "m²", "метр")):
+        return round(parsed * 10.7639, 2)
+    return parsed
+
+
+def _extract_size_sqft(item: dict) -> float | None:
+    for key in ("size", "area", "property_size", "built_up_area", "builtup_area", "plot_area"):
+        value = item.get(key)
+        unit = None
+        if isinstance(value, dict):
+            unit = value.get("unit") or value.get("unit_label") or value.get("unitLabel")
+            value = value.get("value") or value.get("amount") or value.get("text")
+        parsed = _area_to_sqft(value, unit)
+        if parsed is not None:
+            return parsed
+
+    for node in _walk(item):
+        if not isinstance(node, dict):
+            continue
+        name = str(node.get("name") or node.get("label") or node.get("key") or "").lower()
+        if not any(token in name for token in ("size", "area", "sqft", "sq ft", "площад")):
+            continue
+        parsed = _area_to_sqft(
+            node.get("value") or node.get("amount") or node.get("text"),
+            str(node.get("unit") or node.get("unit_label") or ""),
+        )
+        if parsed is not None:
+            return parsed
+    return None
+
+
+def _location_candidate(node: dict) -> tuple[int, str] | None:
+    rank = _LOC_TYPE_PRIORITY.get(str(node.get("type", "")).upper(), -1)
+    name = str(node.get("name") or "").strip()
+    if rank < 0 or not name:
+        return None
+    return rank, name
+
+
+def _extract_building_from(node) -> str | None:
+    best_name: str | None = None
+    best_rank = -1
+    for item in _walk(node):
+        if not isinstance(item, dict):
+            continue
+        candidate = _location_candidate(item)
+        if not candidate:
+            continue
+        rank, name = candidate
+        if rank > best_rank:
+            best_rank, best_name = rank, name
+    return best_name
+
+
+def _extract_building(data: dict, item: dict) -> str | None:
+    for key in ("location", "location_tree", "locations", "locationTree", "community"):
+        value = item.get(key)
+        if value:
+            building = _extract_building_from(value)
+            if building:
+                return building
+    return _extract_building_from(data)
+
+
 def _find_permit_on_page(data: dict) -> str | None:
     """The DLD permit number lives in a regulatory block rendered as an image,
     but its plain value is still in __NEXT_DATA__: the dict that carries a
@@ -119,6 +223,15 @@ def _extract_id_from_url(url: str) -> str | None:
     return m.group(1) if m else None
 
 
+def is_listing_url(url: str) -> bool:
+    """True only for a concrete PF listing URL.
+
+    PF search pages also contain listing-like JSON. Treating them as a detail
+    page can bind monitoring to a random result, so callers must reject them.
+    """
+    return bool(_extract_id_from_url(url or ""))
+
+
 def _is_listing_dict(item: dict) -> bool:
     """Heuristic: a listing dict contains a price plus an id-like field."""
     if not isinstance(item, dict):
@@ -131,12 +244,22 @@ def _is_listing_dict(item: dict) -> bool:
 class PropertyFinderScraper:
     source = SOURCE
 
+    def is_listing_url(self, url: str) -> bool:
+        return is_listing_url(url)
+
+    def listing_id_from_url(self, url: str) -> str | None:
+        return _extract_id_from_url(url)
+
     def fetch_listing(self, url: str) -> ScrapedListing | None:
         """Refetch a known listing URL. Returns:
         - ScrapedListing(is_active=False) if the URL returns 404 (listing removed)
         - ScrapedListing with current data if alive
         - None on network/parse failure (we won't update the DB in that case)
         """
+        if not is_listing_url(url):
+            logger.warning("PF fetch_listing rejected non-listing URL: %s", url)
+            return None
+
         try:
             html = fetch_html(url)
         except ScraperError as e:
@@ -193,12 +316,18 @@ class PropertyFinderScraper:
             permit_number=_find_permit_on_page(data) or _extract_permit(best),
             agent_name=agent_name,
             agency_name=agency_name,
+            building=_extract_building(data, best),
+            bedrooms=_extract_bedrooms(best),
+            size_sqft=_extract_size_sqft(best),
             is_active=True,
         )
 
     def get_permit(self, url: str) -> str | None:
         """Fetch a listing page and return only its DLD permit number (or None).
         Used to compare candidates against our own permit during suggestions."""
+        if not is_listing_url(url):
+            logger.warning("PF get_permit rejected non-listing URL: %s", url)
+            return None
         try:
             html = fetch_html(url)
         except ScraperError as e:
@@ -300,6 +429,8 @@ class PropertyFinderScraper:
                 agent_name, agency_name = _extract_broker(node)
                 share = node.get("share_url") or node.get("path")
                 cand_url = share if str(share).startswith("http") else urljoin(BASE_URL, str(share or ""))
+                if not is_listing_url(cand_url):
+                    continue
 
                 results.append(
                     ScrapedListing(
@@ -312,6 +443,9 @@ class PropertyFinderScraper:
                         permit_number=_extract_permit(node),
                         agent_name=agent_name,
                         agency_name=agency_name,
+                        building=_extract_building(data, node),
+                        bedrooms=_extract_bedrooms(node),
+                        size_sqft=_extract_size_sqft(node),
                         is_active=True,
                     )
                 )

app/services/monitor.py

@@ -14,7 +14,6 @@ Adding new competitors is done via the web UI (user pastes URLs) — not here.
 from __future__ import annotations
 
 import logging
-import re
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
 
@@ -40,6 +39,7 @@ BAYUT = BayutScraper()
 # Same-building suggestions beyond exact permit matches are a browse heuristic —
 # cap how many we show so the page stays usable.
 _SUGGEST_OTHERS_LIMIT = 30
+_PERMIT_MISSING_DELETE_THRESHOLD = 3
 
 # Bayut moved to fully client-side rendering (no __NEXT_DATA__, Algolia keys
 # hidden), so it can't be scraped over plain HTTP — disabled until we add a
@@ -60,6 +60,12 @@ def detect_source_from_url(url: str) -> Source | None:
     return None
 
 
+def _is_supported_listing_url(source: Source, url: str) -> bool:
+    if source == Source.PROPERTYFINDER:
+        return PF.is_listing_url(url)
+    return source == Source.BAYUT
+
+
 def _fmt_price(value: float | None, currency: str | None = "AED") -> str:
     if value is None:
         return "—"
@@ -79,6 +85,21 @@ def _listing_key(source: Source | str, external_id: str) -> tuple[str, str]:
     return source_value, str(external_id or "")
 
 
+def _project_own_listing_key(project: Project) -> tuple[str, str] | None:
+    if not project.our_url:
+        return None
+    source = detect_source_from_url(project.our_url)
+    if source == Source.PROPERTYFINDER:
+        listing_id = PF.listing_id_from_url(project.our_url)
+        return _listing_key(source, listing_id) if listing_id else None
+    return None
+
+
+def _is_own_listing(project: Project, item: ScrapedListing) -> bool:
+    own_key = _project_own_listing_key(project)
+    return bool(own_key and own_key == _listing_key(item.source, item.external_id))
+
+
 def _format_listing_added(project: Project, listing: CompetitorListing, *, auto: bool) -> str:
     title = listing.title or "без названия"
     prefix = "✅ <b>Автоматически добавлен конкурент</b>" if auto else "✅ <b>Добавлен конкурент</b>"
@@ -118,6 +139,8 @@ def add_competitor_url(db: Session, project: Project, url: str) -> tuple[Competi
             "Bayut временно не поддерживается — площадка перешла на защищённый "
             "рендеринг. Добавляйте ссылки PropertyFinder."
         )
+    if not _is_supported_listing_url(source, url):
+        return None, "Укажите ссылку на конкретное объявление, а не на страницу поиска"
 
     scraper = _scraper_for(source)
     scraped = scraper.fetch_listing(url)
@@ -138,6 +161,8 @@ def add_competitor_url(db: Session, project: Project, url: str) -> tuple[Competi
     )
     if existing:
         return None, "Это объявление уже добавлено в проект"
+    if _is_own_listing(project, scraped):
+        return None, "Это ссылка на наш объект, а не на конкурента"
 
     now = datetime.utcnow()
     listing = CompetitorListing(
@@ -164,6 +189,43 @@ def add_competitor_url(db: Session, project: Project, url: str) -> tuple[Competi
     return listing, ""
 
 
+def parse_our_listing_url(url: str) -> dict:
+    """Parse our own PF listing for project metadata.
+
+    Used by the Go API before project validation, so users can paste only the
+    concrete object URL and let the service fill price/permit/building/area.
+    """
+    url = (url or "").strip()
+    if not url:
+        raise ValueError("URL пустой")
+    source = detect_source_from_url(url)
+    if source is None:
+        raise ValueError("URL должен быть с propertyfinder.ae или bayut.com")
+    if source == Source.BAYUT and not BAYUT_ENABLED:
+        raise ValueError(
+            "Bayut временно не поддерживается — площадка перешла на защищённый "
+            "рендеринг. Используйте ссылку PropertyFinder."
+        )
+    if not _is_supported_listing_url(source, url):
+        raise ValueError("Укажите ссылку на конкретное объявление, а не на страницу поиска")
+
+    scraped = _scraper_for(source).fetch_listing(url)
+    if scraped is None:
+        raise ValueError("Не удалось загрузить страницу — сайт мог заблокировать запрос, попробуйте позже")
+    if not scraped.is_active:
+        raise ValueError("Страница объявления вернула 404 — ссылка битая или объявление снято")
+
+    return {
+        "title": scraped.title,
+        "our_price": scraped.price,
+        "dld_permit": scraped.permit_number,
+        "building": scraped.building,
+        "bedrooms": scraped.bedrooms,
+        "size_sqft": scraped.size_sqft,
+        "currency": scraped.currency or "AED",
+    }
+
+
 def add_competitor_urls(db: Session, project: Project, urls: list[str]) -> dict:
     """Add several pasted/selected URLs in one go (used by the suggest page's
     multi-select). Processes them sequentially — each one re-fetches the page —
@@ -237,12 +299,14 @@ def _hide_tracked_suggestions(
 def sync_permit_competitors(
     db: Session,
     project: Project,
+    *,
+    count_missing: bool = True,
 ) -> tuple[list[str], dict[str, list[ScrapedListing]], str | None]:
     """Auto-maintain competitor listings with the same DLD permit.
 
     Exact-permit matches are added automatically. Previously auto-discovered
-    exact-permit listings that disappear from the next permit search are
-    deleted. Manual competitors are never auto-deleted.
+    exact-permit listings are deleted only after several consecutive permit
+    searches miss them. Manual competitors are never auto-deleted.
     """
     changes: list[str] = []
     our_permit = resolve_our_permit(project)
@@ -255,6 +319,7 @@ def sync_permit_competitors(
         item
         for item in suggestions["propertyfinder"]
         if _normalize_permit(item.permit_number) == normalized_permit
+        and not _is_own_listing(project, item)
     ]
 
     matched_keys = {_listing_key(item.source, item.external_id) for item in matches}
@@ -268,6 +333,7 @@ def sync_permit_competitors(
         listing = existing.get(key)
         if listing:
             listing.permit_number = item.permit_number or our_permit
+            listing.permit_missing_checks = 0
             if item.title:
                 listing.title = item.title
             if item.agent_name:
@@ -287,6 +353,11 @@ def sync_permit_competitors(
             continue
         if _listing_key(listing.source, listing.external_id) in matched_keys:
             continue
+        if not count_missing:
+            continue
+        listing.permit_missing_checks = (listing.permit_missing_checks or 0) + 1
+        if listing.permit_missing_checks < _PERMIT_MISSING_DELETE_THRESHOLD:
+            continue
         changes.append(_format_listing_removed(project, listing, auto=True))
         db.delete(listing)
 
@@ -297,6 +368,7 @@ def check_project(db: Session, project: Project) -> list[str]:
     """Re-scan all tracked competitor listings for one project. Returns notification texts."""
     changes: list[str] = []
     now = datetime.utcnow()
+    changes.extend(refresh_our_listing(db, project, now=now))
     sync_changes, _, _ = sync_permit_competitors(db, project)
     changes.extend(sync_changes)
 
@@ -363,6 +435,58 @@ def check_project(db: Session, project: Project) -> list[str]:
     return changes
 
 
+def refresh_our_listing(db: Session, project: Project, *, now: datetime | None = None) -> list[str]:
+    """Parse our own listing and keep project.our_price in sync.
+
+    This never creates a competitor listing. It only updates project metadata
+    from the concrete `our_url`, so PF search pages are ignored.
+    """
+    url = (project.our_url or "").strip()
+    if not url:
+        return []
+    source = detect_source_from_url(url)
+    if source is None or source == Source.BAYUT and not BAYUT_ENABLED:
+        return []
+    if not _is_supported_listing_url(source, url):
+        logger.warning("Project %s has non-listing our_url: %s", project.id, url)
+        return []
+
+    scraped = _scraper_for(source).fetch_listing(url)
+    if scraped is None or not scraped.is_active:
+        return []
+
+    changed: list[str] = []
+    if scraped.permit_number and not project.dld_permit:
+        project.dld_permit = scraped.permit_number
+    if scraped.building and not project.building:
+        project.building = scraped.building
+    if scraped.bedrooms is not None and project.bedrooms is None:
+        project.bedrooms = scraped.bedrooms
+    if scraped.size_sqft is not None and project.size_sqft is None:
+        project.size_sqft = scraped.size_sqft
+
+    old_price = project.our_price
+    new_price = scraped.price
+    if new_price is not None and old_price != new_price:
+        project.our_price = new_price
+        if old_price is not None:
+            delta = new_price - old_price
+            pct = (delta / old_price * 100.0) if old_price else 0.0
+            arrow = "📈" if delta > 0 else "📉"
+            changed.append(
+                f"{arrow} <b>Наша цена скорректирована</b> — {_source_label(source.value)}\n"
+                f"{project.title}\n"
+                f"Было: {_fmt_price(old_price)}\n"
+                f"Стало: {_fmt_price(new_price, scraped.currency or 'AED')} "
+                f"({'+' if delta > 0 else ''}{delta:,.0f} / {pct:+.1f}%)\n"
+                f"{url}".replace(",", " ")
+            )
+    if now is not None:
+        project.last_checked_at = now
+    db.flush()
+    return changed
+
+
 def _notify_owner(project: Project, changes: list[str]) -> None:
     if not changes:
         return
@@ -454,9 +578,10 @@ def suggest_similar(
         excluded.update((l.source.value, l.external_id) for l in project.listings)
     if project.our_url:
         own_src = detect_source_from_url(project.our_url)
-        m = re.search(r"(\d+)\.html", project.our_url)
-        if own_src and m:
-            excluded.add((own_src.value, m.group(1)))
+        if own_src == Source.PROPERTYFINDER:
+            listing_id = PF.listing_id_from_url(project.our_url)
+            if listing_id:
+                excluded.add((own_src.value, listing_id))
     for src in out:
         out[src] = [c for c in out[src] if (src, c.external_id) not in excluded]
 
@@ -470,8 +595,9 @@ def suggest_similar(
                 permits = list(ex.map(PF.get_permit, [c.url for c in pf]))
             for cand, permit in zip(pf, permits):
                 cand.permit_number = permit
-            matches = [c for c in pf if c.permit_number == our_permit]
-            others = [c for c in pf if c.permit_number != our_permit]
+            normalized = _normalize_permit(our_permit)
+            matches = [c for c in pf if _normalize_permit(c.permit_number) == normalized]
+            others = [c for c in pf if _normalize_permit(c.permit_number) != normalized]
             out["propertyfinder"] = matches + others[:_SUGGEST_OTHERS_LIMIT]
         except Exception as e:
             logger.exception("PF permit enrichment failed: %s", e)

app/worker.py

@@ -10,13 +10,15 @@ import json
 import sys
 from typing import Any
 
+from sqlalchemy import text
+
 from app.db import SessionLocal, init_db
 from app.models import Project
 from app.services.monitor import (
     BAYUT_ENABLED,
     add_competitor_url,
-    _format_listing_added,
     notify_project_changes,
+    parse_our_listing_url,
     run_check_all,
     run_check_for_project,
     sync_permit_competitors,
@@ -66,7 +68,6 @@ def cmd_add_listing(payload: dict[str, Any]) -> None:
         listing, err = add_competitor_url(db, project, url)
         if err:
             _fail(err)
-        notify_project_changes(project, [_format_listing_added(project, listing, auto=False)])
         _write({"listing_id": listing.id})
     finally:
         db.close()
@@ -83,7 +84,6 @@ def cmd_add_listings(payload: dict[str, Any]) -> None:
         added = 0
         skipped = 0
         errors: list[str] = []
-        notifications: list[str] = []
         seen: set[str] = set()
         for raw in urls:
             url = str(raw or "").strip()
@@ -97,9 +97,6 @@ def cmd_add_listings(payload: dict[str, Any]) -> None:
                 errors.append(err)
             else:
                 added += 1
-                notifications.append(_format_listing_added(project, listing, auto=False))
-        if notifications:
-            notify_project_changes(project, notifications)
         _write({"added": added, "skipped": skipped, "errors": errors})
     finally:
         db.close()
@@ -122,7 +119,7 @@ def cmd_suggest(payload: dict[str, Any]) -> None:
         project = db.get(Project, project_id)
         if not project:
             _fail("project not found")
-        changes, suggestions, permit = sync_permit_competitors(db, project)
+        changes, suggestions, permit = sync_permit_competitors(db, project, count_missing=False)
         db.commit()
         if changes:
             notify_project_changes(project, changes)
@@ -138,12 +135,31 @@ def cmd_suggest(payload: dict[str, Any]) -> None:
         db.close()
 
 
+def cmd_parse_own_listing(payload: dict[str, Any]) -> None:
+    url = str(payload.get("url") or "")
+    try:
+        _write(parse_our_listing_url(url))
+    except ValueError as exc:
+        _fail(str(exc))
+
+
+def cmd_health(_: dict[str, Any]) -> None:
+    db = SessionLocal()
+    try:
+        db.execute(text("SELECT 1"))
+        _write({"status": "ok"})
+    finally:
+        db.close()
+
+
 COMMANDS = {
+    "health": cmd_health,
     "add-listing": cmd_add_listing,
     "add-listings": cmd_add_listings,
     "check-project": cmd_check_project,
     "check-all": cmd_check_all,
     "suggest": cmd_suggest,
+    "parse-own-listing": cmd_parse_own_listing,
 }
 
 

cmd/bot/main.go

@@ -29,7 +29,11 @@ func main() {
 		slog.Error("db_open_failed", "error", err)
 		os.Exit(1)
 	}
-	defer app.Close()
+	defer func() {
+		if err := app.Close(); err != nil {
+			slog.Warn("app_close_failed", "error", err)
+		}
+	}()
 	if !app.TG.Enabled() {
 		slog.Error("telegram_token_missing")
 		os.Exit(1)

cmd/scheduler/main.go

@@ -24,7 +24,11 @@ func main() {
 		slog.Error("db_open_failed", "error", err)
 		os.Exit(1)
 	}
-	defer app.Close()
+	defer func() {
+		if err := app.Close(); err != nil {
+			slog.Warn("app_close_failed", "error", err)
+		}
+	}()
 
 	interval := cfg.SchedulerInterval()
 	slog.Info("monitoring_pf_scheduler_started", "interval", interval.String())

cmd/server/main.go

@@ -27,7 +27,11 @@ func main() {
 		slog.Error("db_open_failed", "error", err)
 		os.Exit(1)
 	}
-	defer app.Close()
+	defer func() {
+		if err := app.Close(); err != nil {
+			slog.Warn("app_close_failed", "error", err)
+		}
+	}()
 
 	server := &http.Server{
 		Addr:              fmt.Sprintf("%s:%d", cfg.WebHost, cfg.WebPort),

go.mod

@@ -1,8 +1,11 @@
 module monitoring-pf
 
-go 1.25.0
+go 1.25.7
 
-require modernc.org/sqlite v1.50.1
+require (
+	gitea.estateliga.work/admin/portal-common v0.3.0
+	modernc.org/sqlite v1.50.1
+)
 
 require (
 	github.com/dustin/go-humanize v1.0.1 // indirect

go.sum

@@ -1,3 +1,5 @@
+gitea.estateliga.work/admin/portal-common v0.3.0 h1:xpr9UeLXk5pCcNXcTVGZzJZr0Ni7An7DV0OkuYv9qVM=
+gitea.estateliga.work/admin/portal-common v0.3.0/go.mod h1:C860q6g38KVMsv+mKv6k1Vm7smVRCycl+N6r63TElnk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=

internal/pf/config.go

@@ -15,6 +15,7 @@ type Config struct {
 	ScrapeIntervalHours int
 	TGBotToken          string
 	TGBotUsername       string
+	InternalAPIKey      string
 	WorkerPython        string
 	WorkerModule        string
 }
@@ -28,6 +29,7 @@ func LoadConfig() Config {
 		ScrapeIntervalHours: max(1, envInt("SCRAPE_INTERVAL_HOURS", 4)),
 		TGBotToken:          env("TG_BOT_TOKEN", ""),
 		TGBotUsername:       strings.TrimPrefix(env("TG_BOT_USERNAME", ""), "@"),
+		InternalAPIKey:      env("INTERNAL_API_KEY", env("PORTAL_INTERNAL_API_KEY", "")),
 		WorkerPython:        env("WORKER_PYTHON", "python"),
 		WorkerModule:        env("WORKER_MODULE", "app.worker"),
 	}

internal/pf/db.go

@@ -76,6 +76,22 @@ type PricePoint struct {
 	RecordedAt *string  `json:"recorded_at"`
 }
 
+type TeamOverviewRow struct {
+	EmployeeID         int64    `json:"employee_id"`
+	EmployeeName       string   `json:"employee_name"`
+	PortalUserID       *string  `json:"portal_user_id"`
+	TelegramLinked     bool     `json:"telegram_linked"`
+	ProjectID          *int64   `json:"project_id"`
+	ProjectTitle       *string  `json:"project_title"`
+	DealType           *string  `json:"deal_type"`
+	DLDPermit          *string  `json:"dld_permit"`
+	LastCheckedAt      *string  `json:"last_checked_at"`
+	ListingsTotal      int64    `json:"listings_total"`
+	ListingsActive     int64    `json:"listings_active"`
+	ListingsRemoved    int64    `json:"listings_removed"`
+	MinCompetitorPrice *float64 `json:"min_competitor_price"`
+}
+
 func OpenApp(ctx context.Context, cfg Config) (*App, error) {
 	db, err := sql.Open("sqlite", sqliteDSN(cfg.DatabaseURL))
 	if err != nil {
@@ -121,6 +137,14 @@ func (a *App) Close() error {
 	return a.DB.Close()
 }
 
+func closeRows(rows *sql.Rows) {
+	_ = rows.Close()
+}
+
+func rollbackTx(tx *sql.Tx) {
+	_ = tx.Rollback()
+}
+
 func (a *App) InitDB(ctx context.Context) error {
 	stmts := []string{
 		`CREATE TABLE IF NOT EXISTS employees (
@@ -161,6 +185,7 @@ func (a *App) InitDB(ctx context.Context) error {
 			agency_name VARCHAR(300),
 			permit_number VARCHAR(100),
 			auto_discovered BOOLEAN NOT NULL DEFAULT 0,
+			permit_missing_checks INTEGER NOT NULL DEFAULT 0,
 			current_price FLOAT,
 			currency VARCHAR(10),
 			status VARCHAR(7) NOT NULL,
@@ -193,7 +218,7 @@ func (a *App) migrateEmployees(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	defer rows.Close()
+	defer closeRows(rows)
 	columns := map[string]bool{}
 	for rows.Next() {
 		var cid int
@@ -220,7 +245,7 @@ func (a *App) migrateCompetitorListings(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	defer rows.Close()
+	defer closeRows(rows)
 	columns := map[string]bool{}
 	for rows.Next() {
 		var cid int
@@ -243,6 +268,11 @@ func (a *App) migrateCompetitorListings(ctx context.Context) error {
 			return err
 		}
 	}
+	if !columns["permit_missing_checks"] {
+		if _, err := a.DB.ExecContext(ctx, `ALTER TABLE competitor_listings ADD COLUMN permit_missing_checks INTEGER NOT NULL DEFAULT 0`); err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
@@ -303,15 +333,6 @@ func enumStatusOut(value string) string {
 	}
 }
 
-func enumStatusIn(value string) string {
-	switch strings.ToLower(value) {
-	case "removed":
-		return "REMOVED"
-	default:
-		return "ACTIVE"
-	}
-}
-
 func timeOut(raw sql.NullString) *string {
 	if !raw.Valid || strings.TrimSpace(raw.String) == "" {
 		return nil

internal/pf/http.go

@@ -1,11 +1,16 @@
 package pf
 
 import (
+	"context"
+	"database/sql"
 	"encoding/json"
 	"errors"
 	"net/http"
 	"strconv"
 	"strings"
+	"time"
+
+	commonmw "gitea.estateliga.work/admin/portal-common/middleware"
 )
 
 type Server struct {
@@ -20,19 +25,38 @@ type bulkPayload struct {
 	URLs []string `json:"urls"`
 }
 
+type componentProbe struct {
+	Name      string `json:"name"`
+	Status    string `json:"status"`
+	LatencyMs int64  `json:"latency_ms"`
+	Error     string `json:"error,omitempty"`
+}
+
 func (s Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	path := s.apiPath(r.URL.Path)
 	switch {
 	case path == "/healthz":
 		writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
+	case path == "/health/detail":
+		s.healthDetail(w, r)
 	case path == "/":
 		writeJSON(w, http.StatusOK, map[string]string{"service": "monitoring-pf", "ui": "portal", "api": "go"})
 	case !strings.HasPrefix(path, "/api/v1"):
 		writeError(w, http.StatusNotFound, "not found")
+	default:
+		commonmw.InternalAuth(s.App.Cfg.InternalAPIKey)(http.HandlerFunc(s.serveAPI)).ServeHTTP(w, r)
+	}
+}
+
+func (s Server) serveAPI(w http.ResponseWriter, r *http.Request) {
+	path := s.apiPath(r.URL.Path)
+	switch {
 	case path == "/api/v1/access/me" && r.Method == http.MethodGet:
 		s.accessMe(w, r)
 	case path == "/api/v1/summary" && r.Method == http.MethodGet:
 		s.summary(w, r)
+	case path == "/api/v1/team/overview" && r.Method == http.MethodGet:
+		s.teamOverview(w, r)
 	case path == "/api/v1/employees":
 		s.employees(w, r)
 	case strings.HasPrefix(path, "/api/v1/employees/"):
@@ -59,6 +83,104 @@ func (s Server) apiPath(path string) string {
 	return path
 }
 
+func (s Server) healthDetail(w http.ResponseWriter, r *http.Request) {
+	ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
+	defer cancel()
+
+	components := []componentProbe{
+		s.probeDatabase(ctx),
+		s.probeWorker(ctx),
+		s.probeTelegram(ctx),
+		s.probeScheduler(ctx),
+		s.probeProjectIntegrity(ctx),
+	}
+	writeJSON(w, http.StatusOK, map[string]any{"components": components})
+}
+
+func (s Server) probeDatabase(ctx context.Context) componentProbe {
+	start := time.Now()
+	if err := s.App.DB.PingContext(ctx); err != nil {
+		return componentProbe{Name: "sqlite", Status: "down", LatencyMs: time.Since(start).Milliseconds(), Error: err.Error()}
+	}
+	return componentProbe{Name: "sqlite", Status: "ok", LatencyMs: time.Since(start).Milliseconds()}
+}
+
+func (s Server) probeWorker(ctx context.Context) componentProbe {
+	start := time.Now()
+	if err := s.App.Worker.Health(ctx); err != nil {
+		return componentProbe{Name: "python_worker", Status: "down", LatencyMs: time.Since(start).Milliseconds(), Error: err.Error()}
+	}
+	return componentProbe{Name: "python_worker", Status: "ok", LatencyMs: time.Since(start).Milliseconds()}
+}
+
+func (s Server) probeTelegram(ctx context.Context) componentProbe {
+	start := time.Now()
+	if !s.App.TG.Enabled() {
+		return componentProbe{Name: "telegram_bot", Status: "down", LatencyMs: time.Since(start).Milliseconds(), Error: "TG_BOT_TOKEN is not configured"}
+	}
+	if _, err := s.App.TG.BotUsername(ctx); err != nil {
+		return componentProbe{Name: "telegram_bot", Status: "down", LatencyMs: time.Since(start).Milliseconds(), Error: err.Error()}
+	}
+	return componentProbe{Name: "telegram_bot", Status: "ok", LatencyMs: time.Since(start).Milliseconds()}
+}
+
+func (s Server) probeScheduler(ctx context.Context) componentProbe {
+	start := time.Now()
+	staleAfter := max(1, s.App.Cfg.ScrapeIntervalHours*2) * 3600
+	staleModifier := "-" + strconv.Itoa(staleAfter) + " seconds"
+	var total, neverChecked, stale int64
+	err := s.App.DB.QueryRowContext(ctx, `
+		SELECT
+			COUNT(*) AS total,
+			COALESCE(SUM(CASE WHEN last_checked_at IS NULL THEN 1 ELSE 0 END), 0) AS never_checked,
+			COALESCE(SUM(CASE WHEN last_checked_at IS NOT NULL AND datetime(last_checked_at) < datetime('now', ?) THEN 1 ELSE 0 END), 0) AS stale
+		FROM projects`,
+		staleModifier,
+	).Scan(&total, &neverChecked, &stale)
+	if err != nil {
+		return componentProbe{Name: "scheduler", Status: "down", LatencyMs: time.Since(start).Milliseconds(), Error: err.Error()}
+	}
+	if total > 0 && (neverChecked > 0 || stale > 0) {
+		return componentProbe{
+			Name:      "scheduler",
+			Status:    "down",
+			LatencyMs: time.Since(start).Milliseconds(),
+			Error: "projects=" + strconv.FormatInt(total, 10) +
+				" never_checked=" + strconv.FormatInt(neverChecked, 10) +
+				" stale=" + strconv.FormatInt(stale, 10) +
+				" stale_after_sec=" + strconv.Itoa(staleAfter),
+		}
+	}
+	return componentProbe{Name: "scheduler", Status: "ok", LatencyMs: time.Since(start).Milliseconds()}
+}
+
+func (s Server) probeProjectIntegrity(ctx context.Context) componentProbe {
+	start := time.Now()
+	var projects, missingOwner, autoWithoutPermit, searchURLs int64
+	err := s.App.DB.QueryRowContext(ctx, `
+		SELECT
+			(SELECT COUNT(*) FROM projects),
+			(SELECT COUNT(*) FROM projects p LEFT JOIN employees e ON e.id = p.owner_id WHERE e.id IS NULL),
+			(SELECT COUNT(*) FROM competitor_listings WHERE auto_discovered = 1 AND (permit_number IS NULL OR trim(permit_number) = '')),
+			(SELECT COUNT(*) FROM competitor_listings WHERE url LIKE '%/search%' OR url LIKE '%?%' AND external_id = '')
+	`).Scan(&projects, &missingOwner, &autoWithoutPermit, &searchURLs)
+	if err != nil {
+		return componentProbe{Name: "project_integrity", Status: "down", LatencyMs: time.Since(start).Milliseconds(), Error: err.Error()}
+	}
+	if missingOwner > 0 || autoWithoutPermit > 0 || searchURLs > 0 {
+		return componentProbe{
+			Name:      "project_integrity",
+			Status:    "down",
+			LatencyMs: time.Since(start).Milliseconds(),
+			Error: "projects=" + strconv.FormatInt(projects, 10) +
+				" missing_owner=" + strconv.FormatInt(missingOwner, 10) +
+				" auto_without_permit=" + strconv.FormatInt(autoWithoutPermit, 10) +
+				" suspicious_search_urls=" + strconv.FormatInt(searchURLs, 10),
+		}
+	}
+	return componentProbe{Name: "project_integrity", Status: "ok", LatencyMs: time.Since(start).Milliseconds()}
+}
+
 func (s Server) accessMe(w http.ResponseWriter, r *http.Request) {
 	portalID := portalUserID(r)
 	emp, err := s.App.CurrentEmployee(r.Context(), portalID, false)
@@ -86,6 +208,7 @@ func (s Server) accessMe(w http.ResponseWriter, r *http.Request) {
 		"is_admin":               isAdmin(r),
 		"portal_user_id":         nullablePlain(portalID),
 		"telegram_linked":        emp != nil && emp.TGChatID != nil && *emp.TGChatID != "",
+		"can_view_team":          canViewTeam(r),
 		"employee":               emp,
 		"telegram_bot_username":  nullablePlain(botUsername),
 		"telegram_start_command": command,
@@ -94,13 +217,23 @@ func (s Server) accessMe(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s Server) summary(w http.ResponseWriter, r *http.Request) {
-	emp, err := s.App.CurrentEmployee(r.Context(), portalUserID(r), false)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	if emp != nil && emp.TGChatID == nil {
-		emp = nil
+	var emp *Employee
+	var err error
+	if requested := ownerPortalIDFromQuery(r); requested != nil {
+		var ok bool
+		emp, ok = s.resolveProjectOwnerForRead(w, r, requested)
+		if !ok {
+			return
+		}
+	} else {
+		emp, err = s.App.CurrentEmployee(r.Context(), portalUserID(r), false)
+		if err != nil {
+			writeError(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+		if emp != nil && emp.TGChatID == nil {
+			emp = nil
+		}
 	}
 	out, err := s.App.Summary(r.Context(), emp)
 	if err != nil {
@@ -110,6 +243,19 @@ func (s Server) summary(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, out)
 }
 
+func (s Server) teamOverview(w http.ResponseWriter, r *http.Request) {
+	if !canViewTeam(r) {
+		writeError(w, http.StatusNotFound, "not found")
+		return
+	}
+	items, err := s.App.TeamOverview(r.Context(), subordinatePortalIDs(r), isAdmin(r))
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, items)
+}
+
 func (s Server) employees(w http.ResponseWriter, r *http.Request) {
 	switch r.Method {
 	case http.MethodGet:
@@ -118,7 +264,15 @@ func (s Server) employees(w http.ResponseWriter, r *http.Request) {
 			writeError(w, http.StatusInternalServerError, err.Error())
 			return
 		}
-		items, err := s.App.ListEmployees(r.Context(), isAdmin(r), emp)
+		var items []Employee
+		if isAdmin(r) {
+			items, err = s.App.ListEmployees(r.Context(), true, emp)
+		} else if canViewTeam(r) {
+			ids := append(subordinatePortalIDs(r), portalUserID(r))
+			items, err = s.App.ListEmployeesByPortalUserIDs(r.Context(), ids)
+		} else {
+			items, err = s.App.ListEmployees(r.Context(), false, emp)
+		}
 		if err != nil {
 			writeError(w, http.StatusInternalServerError, err.Error())
 			return
@@ -186,12 +340,18 @@ func (s Server) employeeItem(w http.ResponseWriter, r *http.Request, path string
 }
 
 func (s Server) projects(w http.ResponseWriter, r *http.Request) {
-	emp, ok := s.requireEmployee(w, r)
-	if !ok {
-		return
-	}
 	switch r.Method {
 	case http.MethodGet:
+		var emp *Employee
+		var ok bool
+		if requested := ownerPortalIDFromQuery(r); requested != nil {
+			emp, ok = s.resolveProjectOwnerForRead(w, r, requested)
+		} else {
+			emp, ok = s.requireEmployee(w, r)
+		}
+		if !ok {
+			return
+		}
 		items, err := s.App.ListProjects(r.Context(), emp.ID)
 		if err != nil {
 			writeError(w, http.StatusInternalServerError, err.Error())
@@ -203,7 +363,11 @@ func (s Server) projects(w http.ResponseWriter, r *http.Request) {
 		if !decodeJSON(w, r, &payload) {
 			return
 		}
-		project, err := s.App.CreateProject(r.Context(), emp.ID, payload)
+		owner, ok := s.resolveProjectOwner(w, r, payload.OwnerPortalUserID)
+		if !ok {
+			return
+		}
+		project, err := s.App.CreateProject(r.Context(), owner.ID, payload)
 		if err != nil {
 			writeError(w, http.StatusBadRequest, err.Error())
 			return
@@ -215,10 +379,6 @@ func (s Server) projects(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s Server) projectItem(w http.ResponseWriter, r *http.Request, path string) {
-	emp, ok := s.requireEmployee(w, r)
-	if !ok {
-		return
-	}
 	rest := strings.TrimPrefix(path, "/api/v1/projects/")
 	parts := strings.Split(strings.Trim(rest, "/"), "/")
 	if len(parts) == 0 {
@@ -229,13 +389,17 @@ func (s Server) projectItem(w http.ResponseWriter, r *http.Request, path string)
 	if !ok {
 		return
 	}
+	ownerID, ok := s.projectOwnerIDForAccess(w, r, projectID)
+	if !ok {
+		return
+	}
 	if len(parts) == 1 {
-		s.projectCRUD(w, r, emp.ID, projectID)
+		s.projectCRUD(w, r, ownerID, projectID)
 		return
 	}
 	switch {
 	case len(parts) == 2 && parts[1] == "check" && r.Method == http.MethodPost:
-		if _, err := s.App.ProjectByID(r.Context(), emp.ID, projectID, false); err != nil {
+		if _, err := s.App.ProjectByID(r.Context(), ownerID, projectID, false); err != nil {
 			writeError(w, http.StatusNotFound, "project not found")
 			return
 		}
@@ -246,7 +410,7 @@ func (s Server) projectItem(w http.ResponseWriter, r *http.Request, path string)
 		}
 		writeJSON(w, http.StatusOK, map[string]int{"changes": changes})
 	case len(parts) == 2 && parts[1] == "suggest" && r.Method == http.MethodGet:
-		if _, err := s.App.ProjectByID(r.Context(), emp.ID, projectID, false); err != nil {
+		if _, err := s.App.ProjectByID(r.Context(), ownerID, projectID, false); err != nil {
 			writeError(w, http.StatusNotFound, "project not found")
 			return
 		}
@@ -257,9 +421,9 @@ func (s Server) projectItem(w http.ResponseWriter, r *http.Request, path string)
 		}
 		writeJSON(w, http.StatusOK, out)
 	case len(parts) == 2 && parts[1] == "listings" && r.Method == http.MethodPost:
-		s.addListing(w, r, emp.ID, projectID)
+		s.addListing(w, r, ownerID, projectID)
 	case len(parts) == 3 && parts[1] == "listings" && parts[2] == "bulk" && r.Method == http.MethodPost:
-		s.addListings(w, r, emp.ID, projectID)
+		s.addListings(w, r, ownerID, projectID)
 	default:
 		writeError(w, http.StatusNotFound, "not found")
 	}
@@ -340,10 +504,6 @@ func (s Server) addListings(w http.ResponseWriter, r *http.Request, ownerID, pro
 }
 
 func (s Server) listingItem(w http.ResponseWriter, r *http.Request, path string) {
-	emp, ok := s.requireEmployee(w, r)
-	if !ok {
-		return
-	}
 	id, ok := pathID(w, strings.TrimPrefix(path, "/api/v1/listings/"))
 	if !ok {
 		return
@@ -352,45 +512,17 @@ func (s Server) listingItem(w http.ResponseWriter, r *http.Request, path string)
 		writeError(w, http.StatusMethodNotAllowed, "method not allowed")
 		return
 	}
-	deleted, err := s.App.DeleteListing(r.Context(), emp.ID, id)
-	if err != nil {
+	ownerID, ok := s.listingOwnerIDForAccess(w, r, id)
+	if !ok {
+		return
+	}
+	if err := s.App.DeleteListing(r.Context(), ownerID, id); err != nil {
 		writeError(w, http.StatusNotFound, "listing not found")
 		return
 	}
-	if deleted.OwnerChatID != nil && *deleted.OwnerChatID != "" {
-		_ = s.App.TG.SendMessage(r.Context(), *deleted.OwnerChatID, formatDeletedListingMessage(deleted))
-	}
 	w.WriteHeader(http.StatusNoContent)
 }
 
-func formatDeletedListingMessage(deleted *DeletedListing) string {
-	listing := deleted.Listing
-	title := "без названия"
-	if listing.Title != nil && *listing.Title != "" {
-		title = *listing.Title
-	}
-	price := "—"
-	if listing.CurrentPrice != nil {
-		currency := "AED"
-		if listing.Currency != nil && *listing.Currency != "" {
-			currency = *listing.Currency
-		}
-		price = strconv.FormatFloat(*listing.CurrentPrice, 'f', 0, 64) + " " + currency
-	}
-	permit := "—"
-	if listing.PermitNumber != nil && *listing.PermitNumber != "" {
-		permit = *listing.PermitNumber
-	}
-	return "🏠 <b>" + deleted.ProjectTitle + "</b>\n" +
-		"Тип: " + deleted.ProjectDeal + " · Изменений: 1\n" +
-		"——————————\n" +
-		"🗑️ <b>Удален конкурент</b> — " + listing.Source + "\n" +
-		title + "\n" +
-		"Последняя цена: " + price + "\n" +
-		"Permit: <code>" + permit + "</code>\n" +
-		listing.URL
-}
-
 func (s Server) requireEmployee(w http.ResponseWriter, r *http.Request) (*Employee, bool) {
 	emp, err := s.App.CurrentEmployee(r.Context(), portalUserID(r), true)
 	if errors.Is(err, ErrTelegramRequired) {
@@ -404,12 +536,118 @@ func (s Server) requireEmployee(w http.ResponseWriter, r *http.Request) (*Employ
 	return emp, true
 }
 
+func (s Server) resolveProjectOwner(w http.ResponseWriter, r *http.Request, requested *string) (*Employee, bool) {
+	targetPortalID := strings.TrimSpace(portalUserID(r))
+	if requested != nil && strings.TrimSpace(*requested) != "" {
+		targetPortalID = strings.TrimSpace(*requested)
+	}
+	if targetPortalID == "" {
+		writeError(w, http.StatusForbidden, "Сначала авторизуйтесь в Telegram-боте Monitoring PF")
+		return nil, false
+	}
+	if !canManagePortalUser(r, targetPortalID) {
+		writeError(w, http.StatusForbidden, "Нет прав на создание проекта для этого сотрудника")
+		return nil, false
+	}
+	owner, err := s.App.CurrentEmployee(r.Context(), targetPortalID, true)
+	if errors.Is(err, ErrTelegramRequired) {
+		writeError(w, http.StatusBadRequest, "У владельца проекта не подключен Telegram-бот Monitoring PF")
+		return nil, false
+	}
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return nil, false
+	}
+	return owner, true
+}
+
+func (s Server) resolveProjectOwnerForRead(w http.ResponseWriter, r *http.Request, requested *string) (*Employee, bool) {
+	if requested == nil || strings.TrimSpace(*requested) == "" {
+		return s.requireEmployee(w, r)
+	}
+	targetPortalID := strings.TrimSpace(*requested)
+	if !canManagePortalUser(r, targetPortalID) {
+		writeError(w, http.StatusForbidden, "Нет прав на просмотр объектов этого сотрудника")
+		return nil, false
+	}
+	owner, err := s.App.EmployeeByPortalUserID(r.Context(), targetPortalID)
+	if errors.Is(err, ErrNotFound) || errors.Is(err, sql.ErrNoRows) {
+		writeError(w, http.StatusNotFound, "employee not found")
+		return nil, false
+	}
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return nil, false
+	}
+	return owner, true
+}
+
+func (s Server) projectOwnerIDForAccess(w http.ResponseWriter, r *http.Request, projectID int64) (int64, bool) {
+	owner, err := s.App.ProjectOwner(r.Context(), projectID)
+	if errors.Is(err, ErrNotFound) {
+		writeError(w, http.StatusNotFound, "project not found")
+		return 0, false
+	}
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return 0, false
+	}
+	if owner.PortalUserID == nil || !canManagePortalUser(r, *owner.PortalUserID) {
+		writeError(w, http.StatusNotFound, "project not found")
+		return 0, false
+	}
+	return owner.ID, true
+}
+
+func (s Server) listingOwnerIDForAccess(w http.ResponseWriter, r *http.Request, listingID int64) (int64, bool) {
+	var projectID int64
+	err := s.App.DB.QueryRowContext(r.Context(), `
+		SELECT project_id FROM competitor_listings WHERE id = ?`, listingID).Scan(&projectID)
+	if err != nil {
+		writeError(w, http.StatusNotFound, "listing not found")
+		return 0, false
+	}
+	return s.projectOwnerIDForAccess(w, r, projectID)
+}
+
 func portalUserID(r *http.Request) string {
 	return strings.TrimSpace(r.Header.Get("X-User-Id"))
 }
 
 func isAdmin(r *http.Request) bool {
-	return r.Header.Get("X-User-Is-Admin") == "1"
+	return commonmw.HeaderBool(r, "X-User-Is-Admin")
+}
+
+func canViewTeam(r *http.Request) bool {
+	return isAdmin(r) || commonmw.HeaderBool(r, "X-User-Is-Department-Head")
+}
+
+func canManagePortalUser(r *http.Request, targetPortalID string) bool {
+	targetPortalID = strings.TrimSpace(targetPortalID)
+	if targetPortalID == "" {
+		return false
+	}
+	if targetPortalID == portalUserID(r) || isAdmin(r) {
+		return true
+	}
+	for _, id := range subordinatePortalIDs(r) {
+		if id == targetPortalID {
+			return true
+		}
+	}
+	return false
+}
+
+func subordinatePortalIDs(r *http.Request) []string {
+	return commonmw.HeaderCSV(r, "X-User-Subordinates")
+}
+
+func ownerPortalIDFromQuery(r *http.Request) *string {
+	value := strings.TrimSpace(r.URL.Query().Get("owner_portal_user_id"))
+	if value == "" {
+		return nil
+	}
+	return &value
 }
 
 func nullablePlain(value string) *string {

internal/pf/store.go

@@ -5,11 +5,15 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"regexp"
+	"strings"
 )
 
 var ErrNotFound = errors.New("not found")
 var ErrTelegramRequired = errors.New("telegram required")
 
+var propertyFinderListingURLRe = regexp.MustCompile(`(?i)propertyfinder\.ae/.+-(\d+)\.html(?:[?#].*)?$`)
+
 func (a *App) CurrentEmployee(ctx context.Context, portalUserID string, required bool) (*Employee, error) {
 	if portalUserID == "" {
 		if required {
@@ -56,7 +60,28 @@ func (a *App) ListEmployees(ctx context.Context, isAdmin bool, current *Employee
 	if err != nil {
 		return nil, err
 	}
-	defer rows.Close()
+	defer closeRows(rows)
+	return scanEmployees(rows)
+}
+
+func (a *App) ListEmployeesByPortalUserIDs(ctx context.Context, portalUserIDs []string) ([]Employee, error) {
+	ids := uniqueNonEmpty(portalUserIDs)
+	if len(ids) == 0 {
+		return []Employee{}, nil
+	}
+	args := make([]any, 0, len(ids))
+	placeholders := make([]string, 0, len(ids))
+	for _, id := range ids {
+		args = append(args, id)
+		placeholders = append(placeholders, "?")
+	}
+	rows, err := a.DB.QueryContext(ctx, employeeSelect()+`
+		WHERE e.portal_user_id IN (`+strings.Join(placeholders, ",")+`)
+		ORDER BY e.name COLLATE NOCASE`, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer closeRows(rows)
 	return scanEmployees(rows)
 }
 
@@ -214,16 +239,34 @@ func scanEmployees(rows *sql.Rows) ([]Employee, error) {
 	return items, rows.Err()
 }
 
+func uniqueNonEmpty(values []string) []string {
+	seen := map[string]struct{}{}
+	out := []string{}
+	for _, value := range values {
+		value = strings.TrimSpace(value)
+		if value == "" {
+			continue
+		}
+		if _, ok := seen[value]; ok {
+			continue
+		}
+		seen[value] = struct{}{}
+		out = append(out, value)
+	}
+	return out
+}
+
 type ProjectPayload struct {
-	Title     string   `json:"title"`
-	DealType  string   `json:"deal_type"`
-	OurPrice  *float64 `json:"our_price"`
-	Notes     *string  `json:"notes"`
-	DLDPermit *string  `json:"dld_permit"`
-	Building  *string  `json:"building"`
-	Bedrooms  *int64   `json:"bedrooms"`
-	SizeSqft  *float64 `json:"size_sqft"`
-	OurURL    *string  `json:"our_url"`
+	Title             string   `json:"title"`
+	DealType          string   `json:"deal_type"`
+	OurPrice          *float64 `json:"our_price"`
+	Notes             *string  `json:"notes"`
+	DLDPermit         *string  `json:"dld_permit"`
+	Building          *string  `json:"building"`
+	Bedrooms          *int64   `json:"bedrooms"`
+	SizeSqft          *float64 `json:"size_sqft"`
+	OurURL            *string  `json:"our_url"`
+	OwnerPortalUserID *string  `json:"owner_portal_user_id"`
 }
 
 func (a *App) Summary(ctx context.Context, emp *Employee) (map[string]any, error) {
@@ -267,7 +310,7 @@ func (a *App) ListProjects(ctx context.Context, ownerID int64) ([]Project, error
 	if err != nil {
 		return nil, err
 	}
-	defer rows.Close()
+	defer closeRows(rows)
 	items := []Project{}
 	for rows.Next() {
 		p, err := a.scanProject(ctx, rows, false)
@@ -291,6 +334,17 @@ func (a *App) ProjectByID(ctx context.Context, ownerID, projectID int64, detail
 	return p, nil
 }
 
+func (a *App) ProjectOwner(ctx context.Context, projectID int64) (*Employee, error) {
+	row := a.DB.QueryRowContext(ctx, employeeSelect()+`
+		JOIN projects p ON p.owner_id = e.id
+		WHERE p.id = ?`, projectID)
+	emp, err := scanEmployee(row)
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, ErrNotFound
+	}
+	return emp, err
+}
+
 func (a *App) CreateProject(ctx context.Context, ownerID int64, p ProjectPayload) (*Project, error) {
 	title := cleanString(p.Title)
 	if title == "" {
@@ -300,6 +354,15 @@ func (a *App) CreateProject(ctx context.Context, ownerID int64, p ProjectPayload
 	if err != nil {
 		return nil, err
 	}
+	p.Title = title
+	p.DealType = deal
+	p, err = a.enrichProjectPayloadFromURL(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+	if err := validateProjectRequired(p); err != nil {
+		return nil, err
+	}
 	res, err := a.DB.ExecContext(ctx, `
 		INSERT INTO projects
 		    (title, deal_type, owner_id, our_price, notes, dld_permit, building, bedrooms, size_sqft, our_url, created_at)
@@ -333,6 +396,16 @@ func (a *App) UpdateProject(ctx context.Context, ownerID, projectID int64, p Pro
 			return nil, err
 		}
 	}
+	p = mergeProjectPayload(current, p)
+	p.Title = title
+	p.DealType = deal
+	p, err = a.enrichProjectPayloadFromURL(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+	if err := validateProjectRequired(p); err != nil {
+		return nil, err
+	}
 	_, err = a.DB.ExecContext(ctx, `
 		UPDATE projects
 		   SET title = ?, deal_type = ?, our_price = ?, notes = ?, dld_permit = ?,
@@ -347,12 +420,110 @@ func (a *App) UpdateProject(ctx context.Context, ownerID, projectID int64, p Pro
 	return a.ProjectByID(ctx, ownerID, projectID, true)
 }
 
+func mergeProjectPayload(current *Project, p ProjectPayload) ProjectPayload {
+	if current == nil {
+		return p
+	}
+	if p.OurPrice == nil {
+		p.OurPrice = current.OurPrice
+	}
+	if cleanPtr(p.DLDPermit) == nil {
+		p.DLDPermit = current.DLDPermit
+	}
+	if cleanPtr(p.Building) == nil {
+		p.Building = current.Building
+	}
+	if p.Bedrooms == nil {
+		p.Bedrooms = current.Bedrooms
+	}
+	if p.SizeSqft == nil {
+		p.SizeSqft = current.SizeSqft
+	}
+	if cleanPtr(p.OurURL) == nil {
+		p.OurURL = current.OurURL
+	}
+	return p
+}
+
+func (a *App) enrichProjectPayloadFromURL(ctx context.Context, p ProjectPayload) (ProjectPayload, error) {
+	url := cleanPtr(p.OurURL)
+	if url == nil || a.Worker == nil {
+		return p, nil
+	}
+	parsed, err := a.Worker.ParseOwnListing(ctx, *url)
+	if err != nil {
+		if projectMissingParsedFields(p) {
+			return p, fmt.Errorf("parse our_url: %w", err)
+		}
+		return p, nil
+	}
+	return applyParsedOwnListing(p, parsed), nil
+}
+
+func projectMissingParsedFields(p ProjectPayload) bool {
+	return p.OurPrice == nil ||
+		cleanPtr(p.DLDPermit) == nil ||
+		cleanPtr(p.Building) == nil ||
+		p.Bedrooms == nil ||
+		p.SizeSqft == nil
+}
+
+func applyParsedOwnListing(p ProjectPayload, parsed *ParsedOwnListing) ProjectPayload {
+	if parsed == nil {
+		return p
+	}
+	if parsed.OurPrice != nil && *parsed.OurPrice > 0 {
+		p.OurPrice = parsed.OurPrice
+	}
+	if permit := cleanPtr(parsed.DLDPermit); permit != nil {
+		p.DLDPermit = permit
+	}
+	if building := cleanPtr(parsed.Building); building != nil {
+		p.Building = building
+	}
+	if parsed.Bedrooms != nil {
+		p.Bedrooms = parsed.Bedrooms
+	}
+	if parsed.SizeSqft != nil && *parsed.SizeSqft > 0 {
+		p.SizeSqft = parsed.SizeSqft
+	}
+	return p
+}
+
+func validateProjectRequired(p ProjectPayload) error {
+	if cleanString(p.Title) == "" {
+		return fmt.Errorf("title is required")
+	}
+	if p.OurPrice == nil || *p.OurPrice <= 0 {
+		return fmt.Errorf("our_price is required")
+	}
+	if cleanPtr(p.DLDPermit) == nil {
+		return fmt.Errorf("dld_permit is required")
+	}
+	if cleanPtr(p.Building) == nil {
+		return fmt.Errorf("building is required")
+	}
+	if p.Bedrooms == nil {
+		return fmt.Errorf("bedrooms is required")
+	}
+	if p.SizeSqft == nil || *p.SizeSqft <= 0 {
+		return fmt.Errorf("size_sqft is required")
+	}
+	if cleanPtr(p.OurURL) == nil {
+		return fmt.Errorf("our_url is required")
+	}
+	if !propertyFinderListingURLRe.MatchString(strings.TrimSpace(*p.OurURL)) {
+		return fmt.Errorf("our_url must be a concrete PropertyFinder listing URL")
+	}
+	return nil
+}
+
 func (a *App) DeleteProject(ctx context.Context, ownerID, projectID int64) error {
 	tx, err := a.DB.BeginTx(ctx, nil)
 	if err != nil {
 		return err
 	}
-	defer tx.Rollback()
+	defer rollbackTx(tx)
 	listingRows, err := tx.QueryContext(ctx, `SELECT id FROM competitor_listings WHERE project_id = ?`, projectID)
 	if err != nil {
 		return err
@@ -361,12 +532,12 @@ func (a *App) DeleteProject(ctx context.Context, ownerID, projectID int64) error
 	for listingRows.Next() {
 		var id int64
 		if err := listingRows.Scan(&id); err != nil {
-			listingRows.Close()
+			closeRows(listingRows)
 			return err
 		}
 		listingIDs = append(listingIDs, id)
 	}
-	listingRows.Close()
+	closeRows(listingRows)
 	for _, id := range listingIDs {
 		if _, err := tx.ExecContext(ctx, `DELETE FROM price_history WHERE listing_id = ?`, id); err != nil {
 			return err
@@ -386,53 +557,34 @@ func (a *App) DeleteProject(ctx context.Context, ownerID, projectID int64) error
 	return tx.Commit()
 }
 
-type DeletedListing struct {
-	Listing      *Listing
-	ProjectTitle string
-	ProjectDeal  string
-	OwnerChatID  *string
-}
-
-func (a *App) DeleteListing(ctx context.Context, ownerID, listingID int64) (*DeletedListing, error) {
-	row := a.DB.QueryRowContext(ctx, listingSelect()+`
-		  JOIN projects p ON p.id = l.project_id
-		 WHERE l.id = ? AND p.owner_id = ?`, listingID, ownerID)
-	listing, err := scanListing(row, false)
-	if errors.Is(err, sql.ErrNoRows) {
-		return nil, ErrNotFound
-	}
-	if err != nil {
-		return nil, err
-	}
-	deleted := &DeletedListing{Listing: listing}
-	var deal string
-	var chat sql.NullString
-	if err := a.DB.QueryRowContext(ctx, `
-		SELECT p.title, p.deal_type, e.tg_chat_id
-		  FROM projects p
-		  JOIN employees e ON e.id = p.owner_id
-		 WHERE p.id = ? AND p.owner_id = ?`, listing.ProjectID, ownerID).
-		Scan(&deleted.ProjectTitle, &deal, &chat); err != nil {
-		return nil, err
-	}
-	deleted.ProjectDeal = enumDealOut(deal)
-	deleted.OwnerChatID = nullableString(chat)
-
+func (a *App) DeleteListing(ctx context.Context, ownerID, listingID int64) error {
 	tx, err := a.DB.BeginTx(ctx, nil)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	defer tx.Rollback()
-	if _, err := tx.ExecContext(ctx, `DELETE FROM price_history WHERE listing_id = ?`, listing.ID); err != nil {
-		return nil, err
+	defer rollbackTx(tx)
+	if _, err := tx.ExecContext(ctx, `
+		DELETE FROM price_history
+		 WHERE listing_id IN (
+		       SELECT l.id
+		         FROM competitor_listings l
+		         JOIN projects p ON p.id = l.project_id
+		        WHERE l.id = ? AND p.owner_id = ?
+		 )`, listingID, ownerID); err != nil {
+		return err
 	}
-	if _, err := tx.ExecContext(ctx, `DELETE FROM competitor_listings WHERE id = ?`, listing.ID); err != nil {
-		return nil, err
+	res, err := tx.ExecContext(ctx, `
+		DELETE FROM competitor_listings
+		 WHERE id = ?
+		   AND project_id IN (SELECT id FROM projects WHERE owner_id = ?)`, listingID, ownerID)
+	if err != nil {
+		return err
 	}
-	if err := tx.Commit(); err != nil {
-		return nil, err
+	affected, _ := res.RowsAffected()
+	if affected == 0 {
+		return ErrNotFound
 	}
-	return deleted, nil
+	return tx.Commit()
 }
 
 func (a *App) ListingByID(ctx context.Context, id int64, withHistory bool) (*Listing, error) {
@@ -526,10 +678,10 @@ func (a *App) ListingsForProject(ctx context.Context, projectID int64, withHisto
 		items = append(items, *item)
 	}
 	if err := rows.Err(); err != nil {
-		rows.Close()
+		closeRows(rows)
 		return nil, err
 	}
-	rows.Close()
+	closeRows(rows)
 	if withHistory {
 		for i := range items {
 			history, err := a.PriceHistory(ctx, items[i].ID)
@@ -542,6 +694,87 @@ func (a *App) ListingsForProject(ctx context.Context, projectID int64, withHisto
 	return items, nil
 }
 
+func (a *App) TeamOverview(ctx context.Context, portalUserIDs []string, all bool) ([]TeamOverviewRow, error) {
+	args := []any{}
+	where := ""
+	if !all {
+		if len(portalUserIDs) == 0 {
+			return []TeamOverviewRow{}, nil
+		}
+		placeholders := make([]string, 0, len(portalUserIDs))
+		for _, id := range portalUserIDs {
+			placeholders = append(placeholders, "?")
+			args = append(args, id)
+		}
+		where = "WHERE e.portal_user_id IN (" + strings.Join(placeholders, ",") + ")"
+	}
+	rows, err := a.DB.QueryContext(ctx, `
+		SELECT e.id,
+		       e.name,
+		       e.portal_user_id,
+		       e.tg_chat_id,
+		       p.id,
+		       p.title,
+		       p.deal_type,
+		       p.dld_permit,
+		       p.last_checked_at,
+		       count(l.id),
+		       sum(CASE WHEN l.status IN ('ACTIVE','active') THEN 1 ELSE 0 END),
+		       sum(CASE WHEN l.status IN ('REMOVED','removed') THEN 1 ELSE 0 END),
+		       min(CASE WHEN l.status IN ('ACTIVE','active') THEN l.current_price ELSE NULL END)
+		  FROM employees e
+		  LEFT JOIN projects p ON p.owner_id = e.id
+		  LEFT JOIN competitor_listings l ON l.project_id = p.id
+		`+where+`
+		 GROUP BY e.id, e.name, e.portal_user_id, e.tg_chat_id,
+		          p.id, p.title, p.deal_type, p.dld_permit, p.last_checked_at, p.created_at
+		 ORDER BY e.name COLLATE NOCASE, p.created_at DESC`, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer closeRows(rows)
+	items := []TeamOverviewRow{}
+	for rows.Next() {
+		var item TeamOverviewRow
+		var portalID, chatID, title, deal, permit, checked sql.NullString
+		var projectID, total, active, removed sql.NullInt64
+		var minPrice sql.NullFloat64
+		if err := rows.Scan(
+			&item.EmployeeID,
+			&item.EmployeeName,
+			&portalID,
+			&chatID,
+			&projectID,
+			&title,
+			&deal,
+			&permit,
+			&checked,
+			&total,
+			&active,
+			&removed,
+			&minPrice,
+		); err != nil {
+			return nil, err
+		}
+		item.PortalUserID = nullableString(portalID)
+		item.TelegramLinked = chatID.Valid && chatID.String != ""
+		item.ProjectID = nullableInt(projectID)
+		item.ProjectTitle = nullableString(title)
+		if deal.Valid {
+			value := enumDealOut(deal.String)
+			item.DealType = &value
+		}
+		item.DLDPermit = nullableString(permit)
+		item.LastCheckedAt = timeOut(checked)
+		item.ListingsTotal = nullIntValue(total)
+		item.ListingsActive = nullIntValue(active)
+		item.ListingsRemoved = nullIntValue(removed)
+		item.MinCompetitorPrice = nullableFloat(minPrice)
+		items = append(items, item)
+	}
+	return items, rows.Err()
+}
+
 func (a *App) PriceHistory(ctx context.Context, listingID int64) ([]PricePoint, error) {
 	rows, err := a.DB.QueryContext(ctx, `
 		SELECT id, price, recorded_at
@@ -551,7 +784,7 @@ func (a *App) PriceHistory(ctx context.Context, listingID int64) ([]PricePoint,
 	if err != nil {
 		return nil, err
 	}
-	defer rows.Close()
+	defer closeRows(rows)
 	out := []PricePoint{}
 	for rows.Next() {
 		var p PricePoint

internal/pf/store_test.go

@@ -0,0 +1,105 @@
+package pf
+
+import (
+	"strings"
+	"testing"
+)
+
+func strPtr(v string) *string {
+	return &v
+}
+
+func int64Ptr(v int64) *int64 {
+	return &v
+}
+
+func float64Ptr(v float64) *float64 {
+	return &v
+}
+
+func validProjectPayload() ProjectPayload {
+	return ProjectPayload{
+		Title:     "Full Park View",
+		DealType:  "sale",
+		OurPrice:  float64Ptr(2500000),
+		DLDPermit: strPtr("7140504127"),
+		Building:  strPtr("Harbour Gate Tower 2"),
+		Bedrooms:  int64Ptr(2),
+		SizeSqft:  float64Ptr(1081),
+		OurURL: strPtr(
+			"https://www.propertyfinder.ae/en/plp/buy/apartment-for-sale-dubai-dubai-creek-harbour-the-lagoons-harbour-gate-harbour-gate-tower-2-86176216.html",
+		),
+	}
+}
+
+func TestValidateProjectRequiredAcceptsConcretePropertyFinderListingURL(t *testing.T) {
+	payload := validProjectPayload()
+
+	if err := validateProjectRequired(payload); err != nil {
+		t.Fatalf("validateProjectRequired() returned unexpected error: %v", err)
+	}
+}
+
+func TestValidateProjectRequiredRejectsSearchPageAsOurURL(t *testing.T) {
+	payload := validProjectPayload()
+	payload.OurURL = strPtr("https://www.propertyfinder.ae/en/search?c=1&l=12345")
+
+	err := validateProjectRequired(payload)
+	if err == nil {
+		t.Fatal("validateProjectRequired() accepted a search page as our_url")
+	}
+	if !strings.Contains(err.Error(), "concrete PropertyFinder listing URL") {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
+func TestValidateProjectRequiredRejectsListingLikeURLWithoutID(t *testing.T) {
+	payload := validProjectPayload()
+	payload.OurURL = strPtr("https://www.propertyfinder.ae/en/plp/buy/apartment-for-sale-dubai-dubai-creek-harbour.html")
+
+	err := validateProjectRequired(payload)
+	if err == nil {
+		t.Fatal("validateProjectRequired() accepted a listing-like URL without listing id")
+	}
+	if !strings.Contains(err.Error(), "concrete PropertyFinder listing URL") {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
+func TestApplyParsedOwnListingFillsProjectMetadata(t *testing.T) {
+	payload := ProjectPayload{
+		Title:    "Full Park View",
+		DealType: "sale",
+		OurURL: strPtr(
+			"https://www.propertyfinder.ae/en/plp/buy/apartment-for-sale-dubai-dubai-creek-harbour-the-lagoons-harbour-gate-harbour-gate-tower-2-86176216.html",
+		),
+	}
+	parsed := &ParsedOwnListing{
+		OurPrice:  float64Ptr(3500000),
+		DLDPermit: strPtr("7140504127"),
+		Building:  strPtr("Harbour Gate Tower 2"),
+		Bedrooms:  int64Ptr(2),
+		SizeSqft:  float64Ptr(1081),
+	}
+
+	payload = applyParsedOwnListing(payload, parsed)
+
+	if err := validateProjectRequired(payload); err != nil {
+		t.Fatalf("validateProjectRequired() after parsed metadata returned error: %v", err)
+	}
+	if payload.OurPrice == nil || *payload.OurPrice != 3500000 {
+		t.Fatalf("our_price was not applied: %#v", payload.OurPrice)
+	}
+	if payload.DLDPermit == nil || *payload.DLDPermit != "7140504127" {
+		t.Fatalf("dld_permit was not applied: %#v", payload.DLDPermit)
+	}
+	if payload.Building == nil || *payload.Building != "Harbour Gate Tower 2" {
+		t.Fatalf("building was not applied: %#v", payload.Building)
+	}
+	if payload.Bedrooms == nil || *payload.Bedrooms != 2 {
+		t.Fatalf("bedrooms was not applied: %#v", payload.Bedrooms)
+	}
+	if payload.SizeSqft == nil || *payload.SizeSqft != 1081 {
+		t.Fatalf("size_sqft was not applied: %#v", payload.SizeSqft)
+	}
+}

internal/pf/worker.go

@@ -34,6 +34,10 @@ type CheckResult struct {
 	Changes int `json:"changes"`
 }
 
+type HealthResult struct {
+	Status string `json:"status"`
+}
+
 type Suggestion struct {
 	Source       string   `json:"source"`
 	ExternalID   string   `json:"external_id"`
@@ -47,6 +51,16 @@ type Suggestion struct {
 	IsActive     bool     `json:"is_active"`
 }
 
+type ParsedOwnListing struct {
+	Title     *string  `json:"title"`
+	OurPrice  *float64 `json:"our_price"`
+	DLDPermit *string  `json:"dld_permit"`
+	Building  *string  `json:"building"`
+	Bedrooms  *int64   `json:"bedrooms"`
+	SizeSqft  *float64 `json:"size_sqft"`
+	Currency  *string  `json:"currency"`
+}
+
 type SuggestionsResponse struct {
 	OurPermit    *string `json:"our_permit"`
 	BayutEnabled bool    `json:"bayut_enabled"`
@@ -104,6 +118,25 @@ func (w *Worker) Suggest(ctx context.Context, projectID int64) (*SuggestionsResp
 	return &out, nil
 }
 
+func (w *Worker) ParseOwnListing(ctx context.Context, url string) (*ParsedOwnListing, error) {
+	var out ParsedOwnListing
+	if err := w.call(ctx, "parse-own-listing", map[string]any{"url": url}, &out); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func (w *Worker) Health(ctx context.Context) error {
+	var out HealthResult
+	if err := w.call(ctx, "health", map[string]any{}, &out); err != nil {
+		return err
+	}
+	if out.Status != "ok" {
+		return fmt.Errorf("worker status=%s", out.Status)
+	}
+	return nil
+}
+
 func (w *Worker) call(ctx context.Context, command string, payload any, out any) error {
 	ctx, cancel := context.WithTimeout(ctx, 15*time.Minute)
 	defer cancel()

k8s/secrets.yaml

@@ -5,5 +5,6 @@ metadata:
   namespace: monitoring-pf
 type: Opaque
 stringData:
+  INTERNAL_API_KEY: "36fe89ed40c01fdc54d3cf4e3fcacc8751dc456a4a1acd394e9fed48257c5734"
   TG_BOT_TOKEN: "8942895371:AAGCWTr8g0FeqdM3QWmbV_3PxoSb5c_urf0"
   ADMIN_CHAT_ID: ""

tests/test_monitoring_rules.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+import unittest
+from unittest.mock import patch
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from app.db import Base
+from app.models import CompetitorListing, DealType, Employee, ListingStatus, Project, Source
+from app.scrapers.base import ScrapedListing
+from app.scrapers.propertyfinder import PropertyFinderScraper
+from app.services import monitor
+
+
+PF_OWN_URL = (
+    "https://www.propertyfinder.ae/en/plp/buy/apartment-for-sale-dubai-dubai-creek-harbour-"
+    "the-lagoons-harbour-gate-harbour-gate-tower-2-86176216.html"
+)
+PF_COMPETITOR_URL = (
+    "https://www.propertyfinder.ae/en/plp/buy/apartment-for-sale-dubai-dubai-creek-harbour-"
+    "the-lagoons-harbour-gate-harbour-gate-tower-2-86170000.html"
+)
+
+
+def _listing(external_id: str, permit: str | None, url: str = PF_COMPETITOR_URL) -> ScrapedListing:
+    return ScrapedListing(
+        source="propertyfinder",
+        external_id=external_id,
+        url=url,
+        title=f"Listing {external_id}",
+        price=2_500_000,
+        currency="AED",
+        permit_number=permit,
+        agent_name="Agent",
+        agency_name="Agency",
+        is_active=True,
+    )
+
+
+class MonitoringRulesTest(unittest.TestCase):
+    def setUp(self) -> None:
+        engine = create_engine("sqlite:///:memory:", future=True)
+        Base.metadata.create_all(engine)
+        self.Session = sessionmaker(bind=engine, autoflush=False, autocommit=False, future=True)
+        self.db = self.Session()
+
+        owner = Employee(name="Agent", portal_user_id="agent-1", tg_chat_id="100")
+        self.db.add(owner)
+        self.db.flush()
+        self.project = Project(
+            title="Full Park View",
+            deal_type=DealType.SALE,
+            our_price=2_500_000,
+            dld_permit="7140504127",
+            building="Harbour Gate Tower 2",
+            bedrooms=2,
+            size_sqft=1081,
+            our_url=PF_OWN_URL,
+            owner_id=owner.id,
+        )
+        self.db.add(self.project)
+        self.db.commit()
+
+    def tearDown(self) -> None:
+        self.db.close()
+
+    def test_propertyfinder_rejects_search_pages(self) -> None:
+        scraper = PropertyFinderScraper()
+
+        self.assertFalse(scraper.is_listing_url("https://www.propertyfinder.ae/en/search?c=1&l=12345"))
+        self.assertIsNone(scraper.fetch_listing("https://www.propertyfinder.ae/en/search?c=1&l=12345"))
+
+    @patch.object(monitor.PF, "get_permit", side_effect=["7140504127"])
+    @patch.object(
+        monitor.PF,
+        "search_similar",
+        return_value=[
+            _listing("86176216", None, url=PF_OWN_URL),
+            _listing("86170000", None, url=PF_COMPETITOR_URL),
+        ],
+    )
+    def test_suggest_similar_excludes_own_listing(self, _search, _permit) -> None:
+        suggestions = monitor.suggest_similar(self.project, our_permit="7140504127")
+
+        self.assertEqual(["86170000"], [item.external_id for item in suggestions["propertyfinder"]])
+
+    @patch.object(
+        monitor,
+        "suggest_similar",
+        return_value={
+            "propertyfinder": [
+                _listing("86170000", "7140504127"),
+                _listing("86170001", "DIFFERENT"),
+            ],
+            "bayut": [],
+        },
+    )
+    def test_sync_permit_competitors_adds_only_exact_permit_matches(self, _suggest) -> None:
+        changes, suggestions, permit = monitor.sync_permit_competitors(self.db, self.project)
+
+        listings = self.db.query(CompetitorListing).order_by(CompetitorListing.external_id).all()
+        self.assertEqual("7140504127", permit)
+        self.assertEqual(1, len(listings))
+        self.assertEqual("86170000", listings[0].external_id)
+        self.assertTrue(listings[0].auto_discovered)
+        self.assertEqual(["86170001"], [item.external_id for item in suggestions["propertyfinder"]])
+        self.assertEqual(1, len(changes))
+
+    @patch.object(monitor, "suggest_similar", return_value={"propertyfinder": [], "bayut": []})
+    def test_auto_permit_listing_is_removed_only_after_three_misses(self, _suggest) -> None:
+        listing = CompetitorListing(
+            project_id=self.project.id,
+            source=Source.PROPERTYFINDER,
+            external_id="86170000",
+            url=PF_COMPETITOR_URL,
+            title="Competitor",
+            permit_number="7140504127",
+            auto_discovered=True,
+            permit_missing_checks=0,
+            current_price=2_500_000,
+            currency="AED",
+            status=ListingStatus.ACTIVE,
+        )
+        self.db.add(listing)
+        self.db.commit()
+
+        changes, _, _ = monitor.sync_permit_competitors(self.db, self.project)
+        self.db.flush()
+        self.assertEqual([], changes)
+        self.assertEqual(1, self.db.query(CompetitorListing).count())
+        self.assertEqual(1, listing.permit_missing_checks)
+
+        changes, _, _ = monitor.sync_permit_competitors(self.db, self.project)
+        self.db.flush()
+        self.assertEqual([], changes)
+        self.assertEqual(1, self.db.query(CompetitorListing).count())
+        self.assertEqual(2, listing.permit_missing_checks)
+
+        changes, _, _ = monitor.sync_permit_competitors(self.db, self.project)
+        self.db.flush()
+        self.assertEqual(1, len(changes))
+        self.assertEqual(0, self.db.query(CompetitorListing).count())
+
+
+if __name__ == "__main__":
+    unittest.main()