Auto-sync permit competitors in monitoring PF

app/db.py

@@ -30,6 +30,7 @@ def init_db():
 
     Base.metadata.create_all(bind=engine)
     _migrate_employees_portal_user_id()
+    _migrate_competitor_listings_auto_fields()
 
 
 def _migrate_employees_portal_user_id() -> None:
@@ -43,3 +44,15 @@ def _migrate_employees_portal_user_id() -> None:
         conn.execute(
             text("CREATE UNIQUE INDEX IF NOT EXISTS ix_employees_portal_user_id ON employees (portal_user_id)")
         )
+
+
+def _migrate_competitor_listings_auto_fields() -> None:
+    inspector = inspect(engine)
+    if "competitor_listings" not in inspector.get_table_names():
+        return
+    columns = {col["name"] for col in inspector.get_columns("competitor_listings")}
+    with engine.begin() as conn:
+        if "permit_number" not in columns:
+            conn.execute(text("ALTER TABLE competitor_listings ADD COLUMN permit_number VARCHAR(100)"))
+        if "auto_discovered" not in columns:
+            conn.execute(text("ALTER TABLE competitor_listings ADD COLUMN auto_discovered BOOLEAN NOT NULL DEFAULT 0"))

app/models.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from enum import Enum
 
-from sqlalchemy import DateTime, Enum as SAEnum, Float, ForeignKey, Integer, String, Text, UniqueConstraint
+from sqlalchemy import Boolean, DateTime, Enum as SAEnum, Float, ForeignKey, Integer, String, Text, UniqueConstraint
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from app.db import Base
@@ -80,6 +80,8 @@ class CompetitorListing(Base):
     title: Mapped[str | None] = mapped_column(String(500), nullable=True)
     agent_name: Mapped[str | None] = mapped_column(String(300), nullable=True)
     agency_name: Mapped[str | None] = mapped_column(String(300), nullable=True)
+    permit_number: Mapped[str | None] = mapped_column(String(100), nullable=True)
+    auto_discovered: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
 
     current_price: Mapped[float | None] = mapped_column(Float, nullable=True)
     currency: Mapped[str | None] = mapped_column(String(10), nullable=True, default="AED")

app/services/monitor.py

@@ -70,6 +70,39 @@ def _source_label(source: str) -> str:
     return {"propertyfinder": "PropertyFinder", "bayut": "Bayut"}.get(source, source)
 
 
+def _normalize_permit(value: str | None) -> str:
+    return (value or "").strip().lower()
+
+
+def _listing_key(source: Source | str, external_id: str) -> tuple[str, str]:
+    source_value = source.value if isinstance(source, Source) else str(source)
+    return source_value, str(external_id or "")
+
+
+def _format_listing_added(project: Project, listing: CompetitorListing, *, auto: bool) -> str:
+    title = listing.title or "без названия"
+    prefix = "✅ <b>Автоматически добавлен конкурент</b>" if auto else "✅ <b>Добавлен конкурент</b>"
+    return (
+        f"{prefix} — {_source_label(listing.source.value)}\n"
+        f"{title}\n"
+        f"Цена: {_fmt_price(listing.current_price, listing.currency)}\n"
+        f"Permit: <code>{listing.permit_number or project.dld_permit or '—'}</code>\n"
+        f"{listing.url}"
+    )
+
+
+def _format_listing_removed(project: Project, listing: CompetitorListing, *, auto: bool) -> str:
+    title = listing.title or "без названия"
+    prefix = "🗑️ <b>Автоматически удален конкурент</b>" if auto else "🗑️ <b>Удален конкурент</b>"
+    return (
+        f"{prefix} — {_source_label(listing.source.value)}\n"
+        f"{title}\n"
+        f"Последняя цена: {_fmt_price(listing.current_price, listing.currency)}\n"
+        f"Permit: <code>{listing.permit_number or project.dld_permit or '—'}</code>\n"
+        f"{listing.url}"
+    )
+
+
 def add_competitor_url(db: Session, project: Project, url: str) -> tuple[CompetitorListing | None, str]:
     """User-facing entrypoint: paste a URL → create CompetitorListing for the project.
     Returns (listing, error_message). error_message is empty on success.
@@ -115,6 +148,8 @@ def add_competitor_url(db: Session, project: Project, url: str) -> tuple[Competi
         title=scraped.title,
         agent_name=scraped.agent_name,
         agency_name=scraped.agency_name,
+        permit_number=scraped.permit_number,
+        auto_discovered=False,
         current_price=scraped.price,
         currency=scraped.currency or "AED",
         status=ListingStatus.ACTIVE,
@@ -153,12 +188,120 @@ def add_competitor_urls(db: Session, project: Project, urls: list[str]) -> dict:
     return {"added": added, "skipped": skipped, "errors": errors}
 
 
+def _create_listing_from_candidate(
+    db: Session,
+    project: Project,
+    candidate: ScrapedListing,
+    *,
+    permit_number: str,
+) -> CompetitorListing:
+    now = datetime.utcnow()
+    listing = CompetitorListing(
+        project_id=project.id,
+        source=Source(candidate.source),
+        external_id=candidate.external_id or candidate.url,
+        url=candidate.url,
+        title=candidate.title,
+        agent_name=candidate.agent_name,
+        agency_name=candidate.agency_name,
+        permit_number=permit_number,
+        auto_discovered=True,
+        current_price=candidate.price,
+        currency=candidate.currency or "AED",
+        status=ListingStatus.ACTIVE,
+        first_seen_at=now,
+        last_seen_at=now,
+    )
+    db.add(listing)
+    db.flush()
+    if candidate.price is not None:
+        db.add(PriceHistory(listing_id=listing.id, price=candidate.price, recorded_at=now))
+    return listing
+
+
+def _hide_tracked_suggestions(
+    db: Session,
+    project: Project,
+    suggestions: dict[str, list[ScrapedListing]],
+) -> dict[str, list[ScrapedListing]]:
+    tracked = {
+        _listing_key(l.source, l.external_id)
+        for l in db.query(CompetitorListing).filter(CompetitorListing.project_id == project.id).all()
+    }
+    return {
+        source: [item for item in items if _listing_key(item.source, item.external_id) not in tracked]
+        for source, items in suggestions.items()
+    }
+
+
+def sync_permit_competitors(
+    db: Session,
+    project: Project,
+) -> tuple[list[str], dict[str, list[ScrapedListing]], str | None]:
+    """Auto-maintain competitor listings with the same DLD permit.
+
+    Exact-permit matches are added automatically. Previously auto-discovered
+    exact-permit listings that disappear from the next permit search are
+    deleted. Manual competitors are never auto-deleted.
+    """
+    changes: list[str] = []
+    our_permit = resolve_our_permit(project)
+    if not our_permit:
+        return changes, {"propertyfinder": [], "bayut": []}, None
+
+    normalized_permit = _normalize_permit(our_permit)
+    suggestions = suggest_similar(project, our_permit=our_permit, include_tracked=True)
+    matches = [
+        item
+        for item in suggestions["propertyfinder"]
+        if _normalize_permit(item.permit_number) == normalized_permit
+    ]
+
+    matched_keys = {_listing_key(item.source, item.external_id) for item in matches}
+    existing = {
+        _listing_key(item.source, item.external_id): item
+        for item in db.query(CompetitorListing).filter(CompetitorListing.project_id == project.id).all()
+    }
+
+    for item in matches:
+        key = _listing_key(item.source, item.external_id)
+        listing = existing.get(key)
+        if listing:
+            listing.permit_number = item.permit_number or our_permit
+            if item.title:
+                listing.title = item.title
+            if item.agent_name:
+                listing.agent_name = item.agent_name
+            if item.agency_name:
+                listing.agency_name = item.agency_name
+            continue
+
+        listing = _create_listing_from_candidate(db, project, item, permit_number=item.permit_number or our_permit)
+        existing[key] = listing
+        changes.append(_format_listing_added(project, listing, auto=True))
+
+    for listing in list(existing.values()):
+        if not listing.auto_discovered:
+            continue
+        if _normalize_permit(listing.permit_number) != normalized_permit:
+            continue
+        if _listing_key(listing.source, listing.external_id) in matched_keys:
+            continue
+        changes.append(_format_listing_removed(project, listing, auto=True))
+        db.delete(listing)
+
+    return changes, _hide_tracked_suggestions(db, project, suggestions), our_permit
+
+
 def check_project(db: Session, project: Project) -> list[str]:
     """Re-scan all tracked competitor listings for one project. Returns notification texts."""
     changes: list[str] = []
     now = datetime.utcnow()
+    sync_changes, _, _ = sync_permit_competitors(db, project)
+    changes.extend(sync_changes)
 
-    for listing in list(project.listings):
+    listings = db.query(CompetitorListing).filter(CompetitorListing.project_id == project.id).all()
+    for listing in listings:
         scraper = _scraper_for(listing.source)
         scraped = scraper.fetch_listing(listing.url)
         if scraped is None:
@@ -242,6 +385,10 @@ def _notify_owner(project: Project, changes: list[str]) -> None:
         send_message(owner.tg_chat_id, c)
 
 
+def notify_project_changes(project: Project, changes: list[str]) -> None:
+    _notify_owner(project, changes)
+
+
 def _reference_url(project: Project, source: Source) -> str | None:
     """A known listing URL in the project's building for the given source.
 
@@ -268,7 +415,12 @@ def resolve_our_permit(project: Project) -> str | None:
     return PF.get_permit(ref) if ref else None
 
 
-def suggest_similar(project: Project, our_permit: str | None = None) -> dict[str, list[ScrapedListing]]:
+def suggest_similar(
+    project: Project,
+    our_permit: str | None = None,
+    *,
+    include_tracked: bool = False,
+) -> dict[str, list[ScrapedListing]]:
     """Search PF + Bayut for listings in this project's building.
 
     Candidates that share our DLD permit are the same physical listing under a
@@ -295,8 +447,11 @@ def suggest_similar(project: Project, our_permit: str | None = None) -> dict[str
         except Exception as e:
             logger.exception("Bayut suggest failed: %s", e)
 
-    # Hide candidates already tracked for this project — and our own listing.
+    # Hide our own listing. Tracked competitors are hidden for UI suggestions,
-    excluded = {(l.source.value, l.external_id) for l in project.listings}
+    # but included for automatic permit synchronization.
+    excluded: set[tuple[str, str]] = set()
+    if not include_tracked:
+        excluded.update((l.source.value, l.external_id) for l in project.listings)
     if project.our_url:
         own_src = detect_source_from_url(project.our_url)
         m = re.search(r"(\d+)\.html", project.our_url)

app/worker.py

@@ -15,11 +15,11 @@ from app.models import Project
 from app.services.monitor import (
     BAYUT_ENABLED,
     add_competitor_url,
-    add_competitor_urls,
+    _format_listing_added,
-    resolve_our_permit,
+    notify_project_changes,
     run_check_all,
     run_check_for_project,
-    suggest_similar,
+    sync_permit_competitors,
 )
 
 
@@ -66,6 +66,7 @@ def cmd_add_listing(payload: dict[str, Any]) -> None:
         listing, err = add_competitor_url(db, project, url)
         if err:
             _fail(err)
+        notify_project_changes(project, [_format_listing_added(project, listing, auto=False)])
         _write({"listing_id": listing.id})
     finally:
         db.close()
@@ -79,7 +80,27 @@ def cmd_add_listings(payload: dict[str, Any]) -> None:
         project = db.get(Project, project_id)
         if not project:
             _fail("project not found")
-        _write(add_competitor_urls(db, project, urls))
+        added = 0
+        skipped = 0
+        errors: list[str] = []
+        notifications: list[str] = []
+        seen: set[str] = set()
+        for raw in urls:
+            url = str(raw or "").strip()
+            if not url or url in seen:
+                continue
+            seen.add(url)
+            listing, err = add_competitor_url(db, project, url)
+            if err == "Это объявление уже добавлено в проект":
+                skipped += 1
+            elif err:
+                errors.append(err)
+            else:
+                added += 1
+                notifications.append(_format_listing_added(project, listing, auto=False))
+        if notifications:
+            notify_project_changes(project, notifications)
+        _write({"added": added, "skipped": skipped, "errors": errors})
     finally:
         db.close()
 
@@ -101,8 +122,10 @@ def cmd_suggest(payload: dict[str, Any]) -> None:
         project = db.get(Project, project_id)
         if not project:
             _fail("project not found")
-        permit = resolve_our_permit(project)
+        changes, suggestions, permit = sync_permit_competitors(db, project)
-        suggestions = suggest_similar(project, our_permit=permit)
+        db.commit()
+        if changes:
+            notify_project_changes(project, changes)
         _write({
             "our_permit": permit,
             "bayut_enabled": BAYUT_ENABLED,

internal/pf/db.go

@@ -52,20 +52,22 @@ type Project struct {
 }
 
 type Listing struct {
-	ID           int64        `json:"id"`
+	ID             int64        `json:"id"`
-	ProjectID    int64        `json:"project_id"`
+	ProjectID      int64        `json:"project_id"`
-	Source       string       `json:"source"`
+	Source         string       `json:"source"`
-	ExternalID   string       `json:"external_id"`
+	ExternalID     string       `json:"external_id"`
-	URL          string       `json:"url"`
+	URL            string       `json:"url"`
-	Title        *string      `json:"title"`
+	Title          *string      `json:"title"`
-	AgentName    *string      `json:"agent_name"`
+	AgentName      *string      `json:"agent_name"`
-	AgencyName   *string      `json:"agency_name"`
+	AgencyName     *string      `json:"agency_name"`
-	CurrentPrice *float64     `json:"current_price"`
+	PermitNumber   *string      `json:"permit_number"`
-	Currency     *string      `json:"currency"`
+	AutoDiscovered bool         `json:"auto_discovered"`
-	Status       string       `json:"status"`
+	CurrentPrice   *float64     `json:"current_price"`
-	FirstSeenAt  *string      `json:"first_seen_at"`
+	Currency       *string      `json:"currency"`
-	LastSeenAt   *string      `json:"last_seen_at"`
+	Status         string       `json:"status"`
-	PriceHistory []PricePoint `json:"price_history,omitempty"`
+	FirstSeenAt    *string      `json:"first_seen_at"`
+	LastSeenAt     *string      `json:"last_seen_at"`
+	PriceHistory   []PricePoint `json:"price_history,omitempty"`
 }
 
 type PricePoint struct {
@@ -157,6 +159,8 @@ func (a *App) InitDB(ctx context.Context) error {
 			title VARCHAR(500),
 			agent_name VARCHAR(300),
 			agency_name VARCHAR(300),
+			permit_number VARCHAR(100),
+			auto_discovered BOOLEAN NOT NULL DEFAULT 0,
 			current_price FLOAT,
 			currency VARCHAR(10),
 			status VARCHAR(7) NOT NULL,
@@ -178,7 +182,10 @@ func (a *App) InitDB(ctx context.Context) error {
 			return err
 		}
 	}
-	return a.migrateEmployees(ctx)
+	if err := a.migrateEmployees(ctx); err != nil {
+		return err
+	}
+	return a.migrateCompetitorListings(ctx)
 }
 
 func (a *App) migrateEmployees(ctx context.Context) error {
@@ -208,6 +215,37 @@ func (a *App) migrateEmployees(ctx context.Context) error {
 	return err
 }
 
+func (a *App) migrateCompetitorListings(ctx context.Context) error {
+	rows, err := a.DB.QueryContext(ctx, `PRAGMA table_info(competitor_listings)`)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+	columns := map[string]bool{}
+	for rows.Next() {
+		var cid int
+		var name, typ string
+		var notNull int
+		var defaultValue any
+		var pk int
+		if err := rows.Scan(&cid, &name, &typ, &notNull, &defaultValue, &pk); err != nil {
+			return err
+		}
+		columns[name] = true
+	}
+	if !columns["permit_number"] {
+		if _, err := a.DB.ExecContext(ctx, `ALTER TABLE competitor_listings ADD COLUMN permit_number VARCHAR(100)`); err != nil {
+			return err
+		}
+	}
+	if !columns["auto_discovered"] {
+		if _, err := a.DB.ExecContext(ctx, `ALTER TABLE competitor_listings ADD COLUMN auto_discovered BOOLEAN NOT NULL DEFAULT 0`); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func cleanPtr(value *string) *string {
 	if value == nil {
 		return nil

internal/pf/http.go

@@ -352,13 +352,45 @@ func (s Server) listingItem(w http.ResponseWriter, r *http.Request, path string)
 		writeError(w, http.StatusMethodNotAllowed, "method not allowed")
 		return
 	}
-	if err := s.App.DeleteListing(r.Context(), emp.ID, id); err != nil {
+	deleted, err := s.App.DeleteListing(r.Context(), emp.ID, id)
+	if err != nil {
 		writeError(w, http.StatusNotFound, "listing not found")
 		return
 	}
+	if deleted.OwnerChatID != nil && *deleted.OwnerChatID != "" {
+		_ = s.App.TG.SendMessage(r.Context(), *deleted.OwnerChatID, formatDeletedListingMessage(deleted))
+	}
 	w.WriteHeader(http.StatusNoContent)
 }
 
+func formatDeletedListingMessage(deleted *DeletedListing) string {
+	listing := deleted.Listing
+	title := "без названия"
+	if listing.Title != nil && *listing.Title != "" {
+		title = *listing.Title
+	}
+	price := "—"
+	if listing.CurrentPrice != nil {
+		currency := "AED"
+		if listing.Currency != nil && *listing.Currency != "" {
+			currency = *listing.Currency
+		}
+		price = strconv.FormatFloat(*listing.CurrentPrice, 'f', 0, 64) + " " + currency
+	}
+	permit := "—"
+	if listing.PermitNumber != nil && *listing.PermitNumber != "" {
+		permit = *listing.PermitNumber
+	}
+	return "🏠 <b>" + deleted.ProjectTitle + "</b>\n" +
+		"Тип: " + deleted.ProjectDeal + " · Изменений: 1\n" +
+		"——————————\n" +
+		"🗑️ <b>Удален конкурент</b> — " + listing.Source + "\n" +
+		title + "\n" +
+		"Последняя цена: " + price + "\n" +
+		"Permit: <code>" + permit + "</code>\n" +
+		listing.URL
+}
+
 func (s Server) requireEmployee(w http.ResponseWriter, r *http.Request) (*Employee, bool) {
 	emp, err := s.App.CurrentEmployee(r.Context(), portalUserID(r), true)
 	if errors.Is(err, ErrTelegramRequired) {

internal/pf/store.go

@@ -386,30 +386,53 @@ func (a *App) DeleteProject(ctx context.Context, ownerID, projectID int64) error
 	return tx.Commit()
 }
 
-func (a *App) DeleteListing(ctx context.Context, ownerID, listingID int64) error {
+type DeletedListing struct {
-	var id int64
+	Listing      *Listing
-	err := a.DB.QueryRowContext(ctx, `
+	ProjectTitle string
-		SELECT l.id
+	ProjectDeal  string
-		  FROM competitor_listings l JOIN projects p ON p.id = l.project_id
+	OwnerChatID  *string
-		 WHERE l.id = ? AND p.owner_id = ?`, listingID, ownerID).Scan(&id)
+}
+
+func (a *App) DeleteListing(ctx context.Context, ownerID, listingID int64) (*DeletedListing, error) {
+	row := a.DB.QueryRowContext(ctx, listingSelect()+`
+		  JOIN projects p ON p.id = l.project_id
+		 WHERE l.id = ? AND p.owner_id = ?`, listingID, ownerID)
+	listing, err := scanListing(row, false)
 	if errors.Is(err, sql.ErrNoRows) {
-		return ErrNotFound
+		return nil, ErrNotFound
 	}
 	if err != nil {
-		return err
+		return nil, err
 	}
+	deleted := &DeletedListing{Listing: listing}
+	var deal string
+	var chat sql.NullString
+	if err := a.DB.QueryRowContext(ctx, `
+		SELECT p.title, p.deal_type, e.tg_chat_id
+		  FROM projects p
+		  JOIN employees e ON e.id = p.owner_id
+		 WHERE p.id = ? AND p.owner_id = ?`, listing.ProjectID, ownerID).
+		Scan(&deleted.ProjectTitle, &deal, &chat); err != nil {
+		return nil, err
+	}
+	deleted.ProjectDeal = enumDealOut(deal)
+	deleted.OwnerChatID = nullableString(chat)
+
 	tx, err := a.DB.BeginTx(ctx, nil)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	defer tx.Rollback()
-	if _, err := tx.ExecContext(ctx, `DELETE FROM price_history WHERE listing_id = ?`, id); err != nil {
+	if _, err := tx.ExecContext(ctx, `DELETE FROM price_history WHERE listing_id = ?`, listing.ID); err != nil {
-		return err
+		return nil, err
 	}
-	if _, err := tx.ExecContext(ctx, `DELETE FROM competitor_listings WHERE id = ?`, id); err != nil {
+	if _, err := tx.ExecContext(ctx, `DELETE FROM competitor_listings WHERE id = ?`, listing.ID); err != nil {
-		return err
+		return nil, err
 	}
-	return tx.Commit()
+	if err := tx.Commit(); err != nil {
+		return nil, err
+	}
+	return deleted, nil
 }
 
 func (a *App) ListingByID(ctx context.Context, id int64, withHistory bool) (*Listing, error) {
@@ -547,18 +570,19 @@ func (a *App) PriceHistory(ctx context.Context, listingID int64) ([]PricePoint,
 func listingSelect() string {
 	return `
 		SELECT l.id, l.project_id, l.source, l.external_id, l.url, l.title, l.agent_name,
-		       l.agency_name, l.current_price, l.currency, l.status, l.first_seen_at, l.last_seen_at
+		       l.agency_name, l.permit_number, l.auto_discovered, l.current_price, l.currency, l.status, l.first_seen_at, l.last_seen_at
 		  FROM competitor_listings l`
 }
 
 func scanListing(row rowScanner, _ bool) (*Listing, error) {
 	var l Listing
 	var source, status string
-	var title, agent, agency, currency, firstSeen, lastSeen sql.NullString
+	var title, agent, agency, permit, currency, firstSeen, lastSeen sql.NullString
 	var price sql.NullFloat64
+	var autoDiscovered bool
 	if err := row.Scan(
 		&l.ID, &l.ProjectID, &source, &l.ExternalID, &l.URL, &title, &agent, &agency,
-		&price, &currency, &status, &firstSeen, &lastSeen,
+		&permit, &autoDiscovered, &price, &currency, &status, &firstSeen, &lastSeen,
 	); err != nil {
 		return nil, err
 	}
@@ -566,6 +590,8 @@ func scanListing(row rowScanner, _ bool) (*Listing, error) {
 	l.Title = nullableString(title)
 	l.AgentName = nullableString(agent)
 	l.AgencyName = nullableString(agency)
+	l.PermitNumber = nullableString(permit)
+	l.AutoDiscovered = autoDiscovered
 	l.CurrentPrice = nullableFloat(price)
 	l.Currency = nullableString(currency)
 	l.Status = enumStatusOut(status)