Move monitoring PF infrastructure to Go
This commit is contained in:
Grendgi
58
internal/pf/config.go
Normal file
58
internal/pf/config.go
Normal file
@@ -0,0 +1,58 @@
|
||||
package pf
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
WebHost string
|
||||
WebPort int
|
||||
PublicBasePath string
|
||||
DatabaseURL string
|
||||
ScrapeIntervalHours int
|
||||
TGBotToken string
|
||||
TGBotUsername string
|
||||
WorkerPython string
|
||||
WorkerModule string
|
||||
}
|
||||
|
||||
func LoadConfig() Config {
|
||||
return Config{
|
||||
WebHost: env("WEB_HOST", "127.0.0.1"),
|
||||
WebPort: envInt("WEB_PORT", 8000),
|
||||
PublicBasePath: strings.TrimRight(env("PUBLIC_BASE_PATH", ""), "/"),
|
||||
DatabaseURL: env("DATABASE_URL", "sqlite:///data/monitor.db"),
|
||||
ScrapeIntervalHours: max(1, envInt("SCRAPE_INTERVAL_HOURS", 4)),
|
||||
TGBotToken: env("TG_BOT_TOKEN", ""),
|
||||
TGBotUsername: strings.TrimPrefix(env("TG_BOT_USERNAME", ""), "@"),
|
||||
WorkerPython: env("WORKER_PYTHON", "python"),
|
||||
WorkerModule: env("WORKER_MODULE", "app.worker"),
|
||||
}
|
||||
}
|
||||
|
||||
func (c Config) SchedulerInterval() time.Duration {
|
||||
return time.Duration(c.ScrapeIntervalHours) * time.Hour
|
||||
}
|
||||
|
||||
func env(key, fallback string) string {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func envInt(key string, fallback int) int {
|
||||
raw := strings.TrimSpace(os.Getenv(key))
|
||||
if raw == "" {
|
||||
return fallback
|
||||
}
|
||||
value, err := strconv.Atoi(raw)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
316
internal/pf/db.go
Normal file
316
internal/pf/db.go
Normal file
@@ -0,0 +1,316 @@
|
||||
package pf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
type App struct {
|
||||
Cfg Config
|
||||
DB *sql.DB
|
||||
Worker *Worker
|
||||
TG *Telegram
|
||||
}
|
||||
|
||||
type Employee struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
PortalUserID *string `json:"portal_user_id"`
|
||||
TGChatID *string `json:"tg_chat_id"`
|
||||
TGUsername *string `json:"tg_username"`
|
||||
ProjectsTotal int64 `json:"projects_total"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
}
|
||||
|
||||
type Project struct {
|
||||
ID int64 `json:"id"`
|
||||
Title string `json:"title"`
|
||||
DealType string `json:"deal_type"`
|
||||
OurPrice *float64 `json:"our_price"`
|
||||
Notes *string `json:"notes"`
|
||||
DLDPermit *string `json:"dld_permit"`
|
||||
Building *string `json:"building"`
|
||||
Bedrooms *int64 `json:"bedrooms"`
|
||||
SizeSqft *float64 `json:"size_sqft"`
|
||||
OurURL *string `json:"our_url"`
|
||||
OwnerID int64 `json:"owner_id"`
|
||||
Owner *Employee `json:"owner,omitempty"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
LastCheckedAt *string `json:"last_checked_at"`
|
||||
ListingsTotal int64 `json:"listings_total"`
|
||||
ListingsActive int64 `json:"listings_active"`
|
||||
ListingsRemoved int64 `json:"listings_removed"`
|
||||
MinCompetitorPrice *float64 `json:"min_competitor_price"`
|
||||
Listings []Listing `json:"listings,omitempty"`
|
||||
}
|
||||
|
||||
type Listing struct {
|
||||
ID int64 `json:"id"`
|
||||
ProjectID int64 `json:"project_id"`
|
||||
Source string `json:"source"`
|
||||
ExternalID string `json:"external_id"`
|
||||
URL string `json:"url"`
|
||||
Title *string `json:"title"`
|
||||
AgentName *string `json:"agent_name"`
|
||||
AgencyName *string `json:"agency_name"`
|
||||
CurrentPrice *float64 `json:"current_price"`
|
||||
Currency *string `json:"currency"`
|
||||
Status string `json:"status"`
|
||||
FirstSeenAt *string `json:"first_seen_at"`
|
||||
LastSeenAt *string `json:"last_seen_at"`
|
||||
PriceHistory []PricePoint `json:"price_history,omitempty"`
|
||||
}
|
||||
|
||||
type PricePoint struct {
|
||||
ID int64 `json:"id"`
|
||||
Price *float64 `json:"price"`
|
||||
RecordedAt *string `json:"recorded_at"`
|
||||
}
|
||||
|
||||
func OpenApp(ctx context.Context, cfg Config) (*App, error) {
|
||||
db, err := sql.Open("sqlite", sqliteDSN(cfg.DatabaseURL))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, err
|
||||
}
|
||||
app := &App{
|
||||
Cfg: cfg,
|
||||
DB: db,
|
||||
Worker: NewWorker(cfg),
|
||||
TG: NewTelegram(cfg.TGBotToken),
|
||||
}
|
||||
if err := app.InitDB(ctx); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, err
|
||||
}
|
||||
return app, nil
|
||||
}
|
||||
|
||||
func sqliteDSN(databaseURL string) string {
|
||||
const prefix = "sqlite:///"
|
||||
if strings.HasPrefix(databaseURL, prefix) {
|
||||
path := strings.TrimPrefix(databaseURL, prefix)
|
||||
if !strings.HasPrefix(path, "/") {
|
||||
path = filepath.Clean(path)
|
||||
}
|
||||
return path
|
||||
}
|
||||
if strings.HasPrefix(databaseURL, "sqlite://") {
|
||||
u, err := url.Parse(databaseURL)
|
||||
if err == nil && u.Path != "" {
|
||||
return u.Path
|
||||
}
|
||||
}
|
||||
return databaseURL
|
||||
}
|
||||
|
||||
func (a *App) Close() error {
|
||||
return a.DB.Close()
|
||||
}
|
||||
|
||||
func (a *App) InitDB(ctx context.Context) error {
|
||||
stmts := []string{
|
||||
`CREATE TABLE IF NOT EXISTS employees (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name VARCHAR(200) NOT NULL,
|
||||
portal_user_id VARCHAR(100),
|
||||
tg_chat_id VARCHAR(64),
|
||||
tg_username VARCHAR(200),
|
||||
created_at DATETIME
|
||||
)`,
|
||||
`CREATE UNIQUE INDEX IF NOT EXISTS ix_employees_portal_user_id ON employees (portal_user_id)`,
|
||||
`CREATE UNIQUE INDEX IF NOT EXISTS ix_employees_tg_chat_id ON employees (tg_chat_id)`,
|
||||
`CREATE TABLE IF NOT EXISTS projects (
|
||||
id INTEGER PRIMARY KEY,
|
||||
title VARCHAR(300) NOT NULL,
|
||||
deal_type VARCHAR(4) NOT NULL,
|
||||
our_price FLOAT,
|
||||
notes TEXT,
|
||||
dld_permit VARCHAR(100),
|
||||
building VARCHAR(300),
|
||||
bedrooms INTEGER,
|
||||
size_sqft FLOAT,
|
||||
our_url TEXT,
|
||||
owner_id INTEGER NOT NULL,
|
||||
created_at DATETIME,
|
||||
last_checked_at DATETIME,
|
||||
FOREIGN KEY(owner_id) REFERENCES employees(id)
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS ix_projects_dld_permit ON projects (dld_permit)`,
|
||||
`CREATE TABLE IF NOT EXISTS competitor_listings (
|
||||
id INTEGER PRIMARY KEY,
|
||||
project_id INTEGER NOT NULL,
|
||||
source VARCHAR(14) NOT NULL,
|
||||
external_id VARCHAR(100) NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
title VARCHAR(500),
|
||||
agent_name VARCHAR(300),
|
||||
agency_name VARCHAR(300),
|
||||
current_price FLOAT,
|
||||
currency VARCHAR(10),
|
||||
status VARCHAR(7) NOT NULL,
|
||||
first_seen_at DATETIME,
|
||||
last_seen_at DATETIME,
|
||||
FOREIGN KEY(project_id) REFERENCES projects(id)
|
||||
)`,
|
||||
`CREATE UNIQUE INDEX IF NOT EXISTS uq_listing ON competitor_listings (project_id, source, external_id)`,
|
||||
`CREATE TABLE IF NOT EXISTS price_history (
|
||||
id INTEGER PRIMARY KEY,
|
||||
listing_id INTEGER NOT NULL,
|
||||
price FLOAT,
|
||||
recorded_at DATETIME,
|
||||
FOREIGN KEY(listing_id) REFERENCES competitor_listings(id)
|
||||
)`,
|
||||
}
|
||||
for _, stmt := range stmts {
|
||||
if _, err := a.DB.ExecContext(ctx, stmt); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return a.migrateEmployees(ctx)
|
||||
}
|
||||
|
||||
func (a *App) migrateEmployees(ctx context.Context) error {
|
||||
rows, err := a.DB.QueryContext(ctx, `PRAGMA table_info(employees)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
columns := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var name, typ string
|
||||
var notNull int
|
||||
var defaultValue any
|
||||
var pk int
|
||||
if err := rows.Scan(&cid, &name, &typ, ¬Null, &defaultValue, &pk); err != nil {
|
||||
return err
|
||||
}
|
||||
columns[name] = true
|
||||
}
|
||||
if !columns["portal_user_id"] {
|
||||
if _, err := a.DB.ExecContext(ctx, `ALTER TABLE employees ADD COLUMN portal_user_id VARCHAR(100)`); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err = a.DB.ExecContext(ctx, `CREATE UNIQUE INDEX IF NOT EXISTS ix_employees_portal_user_id ON employees (portal_user_id)`)
|
||||
return err
|
||||
}
|
||||
|
||||
func cleanPtr(value *string) *string {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
v := strings.TrimSpace(*value)
|
||||
if v == "" {
|
||||
return nil
|
||||
}
|
||||
return &v
|
||||
}
|
||||
|
||||
func cleanString(value string) string {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func dbNow() string {
|
||||
return time.Now().UTC().Format("2006-01-02 15:04:05.000000")
|
||||
}
|
||||
|
||||
func enumDealIn(value string) (string, error) {
|
||||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||
case "sale", "SALE":
|
||||
return "SALE", nil
|
||||
case "rent", "RENT":
|
||||
return "RENT", nil
|
||||
default:
|
||||
return "", fmt.Errorf("invalid deal_type")
|
||||
}
|
||||
}
|
||||
|
||||
func enumDealOut(value string) string {
|
||||
switch strings.ToUpper(value) {
|
||||
case "RENT":
|
||||
return "rent"
|
||||
default:
|
||||
return "sale"
|
||||
}
|
||||
}
|
||||
|
||||
func enumSourceOut(value string) string {
|
||||
switch strings.ToUpper(value) {
|
||||
case "BAYUT":
|
||||
return "bayut"
|
||||
default:
|
||||
return "propertyfinder"
|
||||
}
|
||||
}
|
||||
|
||||
func enumStatusOut(value string) string {
|
||||
switch strings.ToUpper(value) {
|
||||
case "REMOVED":
|
||||
return "removed"
|
||||
default:
|
||||
return "active"
|
||||
}
|
||||
}
|
||||
|
||||
func enumStatusIn(value string) string {
|
||||
switch strings.ToLower(value) {
|
||||
case "removed":
|
||||
return "REMOVED"
|
||||
default:
|
||||
return "ACTIVE"
|
||||
}
|
||||
}
|
||||
|
||||
func timeOut(raw sql.NullString) *string {
|
||||
if !raw.Valid || strings.TrimSpace(raw.String) == "" {
|
||||
return nil
|
||||
}
|
||||
value := strings.TrimSpace(raw.String)
|
||||
layouts := []string{
|
||||
time.RFC3339Nano,
|
||||
"2006-01-02 15:04:05.999999",
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02T15:04:05.999999",
|
||||
}
|
||||
for _, layout := range layouts {
|
||||
if t, err := time.Parse(layout, value); err == nil {
|
||||
out := t.UTC().Format(time.RFC3339)
|
||||
return &out
|
||||
}
|
||||
}
|
||||
return &value
|
||||
}
|
||||
|
||||
func nullableString(ns sql.NullString) *string {
|
||||
if !ns.Valid {
|
||||
return nil
|
||||
}
|
||||
return &ns.String
|
||||
}
|
||||
|
||||
func nullableFloat(nf sql.NullFloat64) *float64 {
|
||||
if !nf.Valid {
|
||||
return nil
|
||||
}
|
||||
return &nf.Float64
|
||||
}
|
||||
|
||||
func nullableInt(ni sql.NullInt64) *int64 {
|
||||
if !ni.Valid {
|
||||
return nil
|
||||
}
|
||||
return &ni.Int64
|
||||
}
|
||||
414
internal/pf/http.go
Normal file
414
internal/pf/http.go
Normal file
@@ -0,0 +1,414 @@
|
||||
package pf
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
App *App
|
||||
}
|
||||
|
||||
type listingPayload struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type bulkPayload struct {
|
||||
URLs []string `json:"urls"`
|
||||
}
|
||||
|
||||
func (s Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
path := s.apiPath(r.URL.Path)
|
||||
switch {
|
||||
case path == "/healthz":
|
||||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||||
case path == "/":
|
||||
writeJSON(w, http.StatusOK, map[string]string{"service": "monitoring-pf", "ui": "portal", "api": "go"})
|
||||
case !strings.HasPrefix(path, "/api/v1"):
|
||||
writeError(w, http.StatusNotFound, "not found")
|
||||
case path == "/api/v1/access/me" && r.Method == http.MethodGet:
|
||||
s.accessMe(w, r)
|
||||
case path == "/api/v1/summary" && r.Method == http.MethodGet:
|
||||
s.summary(w, r)
|
||||
case path == "/api/v1/employees":
|
||||
s.employees(w, r)
|
||||
case strings.HasPrefix(path, "/api/v1/employees/"):
|
||||
s.employeeItem(w, r, path)
|
||||
case path == "/api/v1/projects":
|
||||
s.projects(w, r)
|
||||
case strings.HasPrefix(path, "/api/v1/projects/"):
|
||||
s.projectItem(w, r, path)
|
||||
case strings.HasPrefix(path, "/api/v1/listings/"):
|
||||
s.listingItem(w, r, path)
|
||||
default:
|
||||
writeError(w, http.StatusNotFound, "not found")
|
||||
}
|
||||
}
|
||||
|
||||
func (s Server) apiPath(path string) string {
|
||||
base := s.App.Cfg.PublicBasePath
|
||||
if base != "" && path == base {
|
||||
return "/"
|
||||
}
|
||||
if base != "" && strings.HasPrefix(path, base+"/") {
|
||||
return strings.TrimPrefix(path, base)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func (s Server) accessMe(w http.ResponseWriter, r *http.Request) {
|
||||
portalID := portalUserID(r)
|
||||
emp, err := s.App.CurrentEmployee(r.Context(), portalID, false)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
var link *string
|
||||
if s.App.Cfg.TGBotUsername != "" && portalID != "" {
|
||||
v := "https://t.me/" + s.App.Cfg.TGBotUsername + "?start=" + portalID
|
||||
link = &v
|
||||
}
|
||||
var command *string
|
||||
if portalID != "" {
|
||||
v := "/start " + portalID
|
||||
command = &v
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"is_admin": isAdmin(r),
|
||||
"portal_user_id": nullablePlain(portalID),
|
||||
"telegram_linked": emp != nil && emp.TGChatID != nil && *emp.TGChatID != "",
|
||||
"employee": emp,
|
||||
"telegram_bot_username": nullablePlain(s.App.Cfg.TGBotUsername),
|
||||
"telegram_start_command": command,
|
||||
"telegram_start_link": link,
|
||||
})
|
||||
}
|
||||
|
||||
func (s Server) summary(w http.ResponseWriter, r *http.Request) {
|
||||
emp, err := s.App.CurrentEmployee(r.Context(), portalUserID(r), false)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if emp != nil && emp.TGChatID == nil {
|
||||
emp = nil
|
||||
}
|
||||
out, err := s.App.Summary(r.Context(), emp)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
func (s Server) employees(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
emp, err := s.App.CurrentEmployee(r.Context(), portalUserID(r), false)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
items, err := s.App.ListEmployees(r.Context(), isAdmin(r), emp)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, items)
|
||||
case http.MethodPost:
|
||||
if !isAdmin(r) {
|
||||
writeError(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
var payload EmployeePayload
|
||||
if !decodeJSON(w, r, &payload) {
|
||||
return
|
||||
}
|
||||
emp, err := s.App.CreateEmployee(r.Context(), payload)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, emp)
|
||||
default:
|
||||
writeError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func (s Server) employeeItem(w http.ResponseWriter, r *http.Request, path string) {
|
||||
id, ok := pathID(w, strings.TrimPrefix(path, "/api/v1/employees/"))
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !isAdmin(r) {
|
||||
writeError(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case http.MethodPatch:
|
||||
var payload EmployeePayload
|
||||
if !decodeJSON(w, r, &payload) {
|
||||
return
|
||||
}
|
||||
emp, err := s.App.UpdateEmployee(r.Context(), id, payload)
|
||||
if err != nil {
|
||||
status := http.StatusBadRequest
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
status = http.StatusNotFound
|
||||
}
|
||||
writeError(w, status, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, emp)
|
||||
case http.MethodDelete:
|
||||
err := s.App.DeleteEmployee(r.Context(), id)
|
||||
if err != nil {
|
||||
status := http.StatusBadRequest
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
status = http.StatusNotFound
|
||||
}
|
||||
writeError(w, status, err.Error())
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
default:
|
||||
writeError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func (s Server) projects(w http.ResponseWriter, r *http.Request) {
|
||||
emp, ok := s.requireEmployee(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
items, err := s.App.ListProjects(r.Context(), emp.ID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, items)
|
||||
case http.MethodPost:
|
||||
var payload ProjectPayload
|
||||
if !decodeJSON(w, r, &payload) {
|
||||
return
|
||||
}
|
||||
project, err := s.App.CreateProject(r.Context(), emp.ID, payload)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, project)
|
||||
default:
|
||||
writeError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func (s Server) projectItem(w http.ResponseWriter, r *http.Request, path string) {
|
||||
emp, ok := s.requireEmployee(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
rest := strings.TrimPrefix(path, "/api/v1/projects/")
|
||||
parts := strings.Split(strings.Trim(rest, "/"), "/")
|
||||
if len(parts) == 0 {
|
||||
writeError(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
projectID, ok := pathID(w, parts[0])
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if len(parts) == 1 {
|
||||
s.projectCRUD(w, r, emp.ID, projectID)
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case len(parts) == 2 && parts[1] == "check" && r.Method == http.MethodPost:
|
||||
if _, err := s.App.ProjectByID(r.Context(), emp.ID, projectID, false); err != nil {
|
||||
writeError(w, http.StatusNotFound, "project not found")
|
||||
return
|
||||
}
|
||||
changes, err := s.App.Worker.CheckProject(r.Context(), projectID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]int{"changes": changes})
|
||||
case len(parts) == 2 && parts[1] == "suggest" && r.Method == http.MethodGet:
|
||||
if _, err := s.App.ProjectByID(r.Context(), emp.ID, projectID, false); err != nil {
|
||||
writeError(w, http.StatusNotFound, "project not found")
|
||||
return
|
||||
}
|
||||
out, err := s.App.Worker.Suggest(r.Context(), projectID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
case len(parts) == 2 && parts[1] == "listings" && r.Method == http.MethodPost:
|
||||
s.addListing(w, r, emp.ID, projectID)
|
||||
case len(parts) == 3 && parts[1] == "listings" && parts[2] == "bulk" && r.Method == http.MethodPost:
|
||||
s.addListings(w, r, emp.ID, projectID)
|
||||
default:
|
||||
writeError(w, http.StatusNotFound, "not found")
|
||||
}
|
||||
}
|
||||
|
||||
func (s Server) projectCRUD(w http.ResponseWriter, r *http.Request, ownerID, projectID int64) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
project, err := s.App.ProjectByID(r.Context(), ownerID, projectID, true)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, "project not found")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, project)
|
||||
case http.MethodPatch:
|
||||
var payload ProjectPayload
|
||||
if !decodeJSON(w, r, &payload) {
|
||||
return
|
||||
}
|
||||
project, err := s.App.UpdateProject(r.Context(), ownerID, projectID, payload)
|
||||
if err != nil {
|
||||
status := http.StatusBadRequest
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
status = http.StatusNotFound
|
||||
}
|
||||
writeError(w, status, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, project)
|
||||
case http.MethodDelete:
|
||||
if err := s.App.DeleteProject(r.Context(), ownerID, projectID); err != nil {
|
||||
writeError(w, http.StatusNotFound, "project not found")
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
default:
|
||||
writeError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func (s Server) addListing(w http.ResponseWriter, r *http.Request, ownerID, projectID int64) {
|
||||
if _, err := s.App.ProjectByID(r.Context(), ownerID, projectID, false); err != nil {
|
||||
writeError(w, http.StatusNotFound, "project not found")
|
||||
return
|
||||
}
|
||||
var payload listingPayload
|
||||
if !decodeJSON(w, r, &payload) {
|
||||
return
|
||||
}
|
||||
id, err := s.App.Worker.AddListing(r.Context(), projectID, payload.URL)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
listing, err := s.App.ListingByID(r.Context(), id, true)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, listing)
|
||||
}
|
||||
|
||||
func (s Server) addListings(w http.ResponseWriter, r *http.Request, ownerID, projectID int64) {
|
||||
if _, err := s.App.ProjectByID(r.Context(), ownerID, projectID, false); err != nil {
|
||||
writeError(w, http.StatusNotFound, "project not found")
|
||||
return
|
||||
}
|
||||
var payload bulkPayload
|
||||
if !decodeJSON(w, r, &payload) {
|
||||
return
|
||||
}
|
||||
out, err := s.App.Worker.AddListings(r.Context(), projectID, payload.URLs)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
func (s Server) listingItem(w http.ResponseWriter, r *http.Request, path string) {
|
||||
emp, ok := s.requireEmployee(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
id, ok := pathID(w, strings.TrimPrefix(path, "/api/v1/listings/"))
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if r.Method != http.MethodDelete {
|
||||
writeError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
if err := s.App.DeleteListing(r.Context(), emp.ID, id); err != nil {
|
||||
writeError(w, http.StatusNotFound, "listing not found")
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (s Server) requireEmployee(w http.ResponseWriter, r *http.Request) (*Employee, bool) {
|
||||
emp, err := s.App.CurrentEmployee(r.Context(), portalUserID(r), true)
|
||||
if errors.Is(err, ErrTelegramRequired) {
|
||||
writeError(w, http.StatusForbidden, "Сначала авторизуйтесь в Telegram-боте Monitoring PF")
|
||||
return nil, false
|
||||
}
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return nil, false
|
||||
}
|
||||
return emp, true
|
||||
}
|
||||
|
||||
func portalUserID(r *http.Request) string {
|
||||
return strings.TrimSpace(r.Header.Get("X-User-Id"))
|
||||
}
|
||||
|
||||
func isAdmin(r *http.Request) bool {
|
||||
return r.Header.Get("X-User-Is-Admin") == "1"
|
||||
}
|
||||
|
||||
func nullablePlain(value string) *string {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return nil
|
||||
}
|
||||
return &value
|
||||
}
|
||||
|
||||
func pathID(w http.ResponseWriter, value string) (int64, bool) {
|
||||
if strings.Contains(value, "/") {
|
||||
writeError(w, http.StatusNotFound, "not found")
|
||||
return 0, false
|
||||
}
|
||||
id, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || id <= 0 {
|
||||
writeError(w, http.StatusNotFound, "not found")
|
||||
return 0, false
|
||||
}
|
||||
return id, true
|
||||
}
|
||||
|
||||
func decodeJSON(w http.ResponseWriter, r *http.Request, out any) bool {
|
||||
defer r.Body.Close()
|
||||
if err := json.NewDecoder(r.Body).Decode(out); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid json")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, value any) {
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, detail string) {
|
||||
writeJSON(w, status, map[string]string{"detail": detail})
|
||||
}
|
||||
569
internal/pf/store.go
Normal file
569
internal/pf/store.go
Normal file
@@ -0,0 +1,569 @@
|
||||
package pf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
var ErrNotFound = errors.New("not found")
|
||||
var ErrTelegramRequired = errors.New("telegram required")
|
||||
|
||||
func (a *App) CurrentEmployee(ctx context.Context, portalUserID string, required bool) (*Employee, error) {
|
||||
if portalUserID == "" {
|
||||
if required {
|
||||
return nil, ErrTelegramRequired
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
emp, err := a.EmployeeByPortalUserID(ctx, portalUserID)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
if required {
|
||||
return nil, ErrTelegramRequired
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if emp.TGChatID == nil || *emp.TGChatID == "" {
|
||||
if required {
|
||||
return nil, ErrTelegramRequired
|
||||
}
|
||||
}
|
||||
return emp, nil
|
||||
}
|
||||
|
||||
func (a *App) EmployeeByPortalUserID(ctx context.Context, portalUserID string) (*Employee, error) {
|
||||
row := a.DB.QueryRowContext(ctx, employeeSelect()+` WHERE e.portal_user_id = ?`, portalUserID)
|
||||
return scanEmployee(row)
|
||||
}
|
||||
|
||||
func (a *App) EmployeeByChatID(ctx context.Context, chatID string) (*Employee, error) {
|
||||
row := a.DB.QueryRowContext(ctx, employeeSelect()+` WHERE e.tg_chat_id = ?`, chatID)
|
||||
return scanEmployee(row)
|
||||
}
|
||||
|
||||
func (a *App) ListEmployees(ctx context.Context, isAdmin bool, current *Employee) ([]Employee, error) {
|
||||
if !isAdmin {
|
||||
if current == nil {
|
||||
return []Employee{}, nil
|
||||
}
|
||||
return []Employee{*current}, nil
|
||||
}
|
||||
rows, err := a.DB.QueryContext(ctx, employeeSelect()+` ORDER BY e.name`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanEmployees(rows)
|
||||
}
|
||||
|
||||
type EmployeePayload struct {
|
||||
Name string `json:"name"`
|
||||
PortalUserID *string `json:"portal_user_id"`
|
||||
TGUsername *string `json:"tg_username"`
|
||||
TGChatID *string `json:"tg_chat_id"`
|
||||
}
|
||||
|
||||
func (a *App) CreateEmployee(ctx context.Context, p EmployeePayload) (*Employee, error) {
|
||||
name := cleanString(p.Name)
|
||||
if name == "" {
|
||||
return nil, fmt.Errorf("name is required")
|
||||
}
|
||||
username := cleanPtr(p.TGUsername)
|
||||
if username != nil && len(*username) > 0 && (*username)[0] == '@' {
|
||||
u := (*username)[1:]
|
||||
username = &u
|
||||
}
|
||||
res, err := a.DB.ExecContext(ctx, `
|
||||
INSERT INTO employees (name, portal_user_id, tg_chat_id, tg_username, created_at)
|
||||
VALUES (?, ?, ?, ?, ?)`,
|
||||
name, cleanPtr(p.PortalUserID), cleanPtr(p.TGChatID), username, dbNow(),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
id, _ := res.LastInsertId()
|
||||
return a.EmployeeByID(ctx, id)
|
||||
}
|
||||
|
||||
func (a *App) EmployeeByID(ctx context.Context, id int64) (*Employee, error) {
|
||||
row := a.DB.QueryRowContext(ctx, employeeSelect()+` WHERE e.id = ?`, id)
|
||||
return scanEmployee(row)
|
||||
}
|
||||
|
||||
func (a *App) UpdateEmployee(ctx context.Context, id int64, p EmployeePayload) (*Employee, error) {
|
||||
emp, err := a.EmployeeByID(ctx, id)
|
||||
if err != nil {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
name := cleanString(p.Name)
|
||||
if name == "" {
|
||||
name = emp.Name
|
||||
}
|
||||
username := cleanPtr(p.TGUsername)
|
||||
if username != nil && len(*username) > 0 && (*username)[0] == '@' {
|
||||
u := (*username)[1:]
|
||||
username = &u
|
||||
}
|
||||
if _, err := a.DB.ExecContext(ctx, `
|
||||
UPDATE employees
|
||||
SET name = ?, portal_user_id = COALESCE(?, portal_user_id), tg_username = ?, tg_chat_id = ?
|
||||
WHERE id = ?`,
|
||||
name, cleanPtr(p.PortalUserID), username, cleanPtr(p.TGChatID), id,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.EmployeeByID(ctx, id)
|
||||
}
|
||||
|
||||
func (a *App) DeleteEmployee(ctx context.Context, id int64) error {
|
||||
var count int64
|
||||
if err := a.DB.QueryRowContext(ctx, `SELECT count(*) FROM projects WHERE owner_id = ?`, id).Scan(&count); err != nil {
|
||||
return err
|
||||
}
|
||||
if count > 0 {
|
||||
return fmt.Errorf("employee has projects")
|
||||
}
|
||||
res, err := a.DB.ExecContext(ctx, `DELETE FROM employees WHERE id = ?`, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, _ := res.RowsAffected()
|
||||
if affected == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) LinkTelegram(ctx context.Context, portalUserID, chatID, username, name string) (*Employee, error) {
|
||||
if existing, err := a.EmployeeByChatID(ctx, chatID); err == nil {
|
||||
if existing.PortalUserID != nil && *existing.PortalUserID != "" && *existing.PortalUserID != portalUserID {
|
||||
return nil, fmt.Errorf("telegram belongs to another portal user")
|
||||
}
|
||||
if existing.PortalUserID == nil || *existing.PortalUserID == "" {
|
||||
if _, err := a.DB.ExecContext(ctx, `
|
||||
UPDATE employees SET portal_user_id = ?, tg_username = ? WHERE id = ?`,
|
||||
portalUserID, nullIfEmpty(username), existing.ID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return a.EmployeeByID(ctx, existing.ID)
|
||||
}
|
||||
if emp, err := a.EmployeeByPortalUserID(ctx, portalUserID); err == nil {
|
||||
if emp.TGChatID != nil && *emp.TGChatID != "" && *emp.TGChatID != chatID {
|
||||
return nil, fmt.Errorf("portal user belongs to another telegram")
|
||||
}
|
||||
_, err := a.DB.ExecContext(ctx, `
|
||||
UPDATE employees SET tg_chat_id = ?, tg_username = ?, name = COALESCE(NULLIF(name, ''), ?) WHERE id = ?`,
|
||||
chatID, nullIfEmpty(username), name, emp.ID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.EmployeeByID(ctx, emp.ID)
|
||||
}
|
||||
res, err := a.DB.ExecContext(ctx, `
|
||||
INSERT INTO employees (name, portal_user_id, tg_chat_id, tg_username, created_at)
|
||||
VALUES (?, ?, ?, ?, ?)`,
|
||||
name, portalUserID, chatID, nullIfEmpty(username), dbNow(),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
id, _ := res.LastInsertId()
|
||||
return a.EmployeeByID(ctx, id)
|
||||
}
|
||||
|
||||
func employeeSelect() string {
|
||||
return `
|
||||
SELECT e.id, e.name, e.portal_user_id, e.tg_chat_id, e.tg_username, e.created_at,
|
||||
(SELECT count(*) FROM projects p WHERE p.owner_id = e.id) AS projects_total
|
||||
FROM employees e`
|
||||
}
|
||||
|
||||
type rowScanner interface {
|
||||
Scan(dest ...any) error
|
||||
}
|
||||
|
||||
func scanEmployee(row rowScanner) (*Employee, error) {
|
||||
var emp Employee
|
||||
var portal, chat, username, created sql.NullString
|
||||
if err := row.Scan(&emp.ID, &emp.Name, &portal, &chat, &username, &created, &emp.ProjectsTotal); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
emp.PortalUserID = nullableString(portal)
|
||||
emp.TGChatID = nullableString(chat)
|
||||
emp.TGUsername = nullableString(username)
|
||||
emp.CreatedAt = timeOut(created)
|
||||
return &emp, nil
|
||||
}
|
||||
|
||||
func scanEmployees(rows *sql.Rows) ([]Employee, error) {
|
||||
items := []Employee{}
|
||||
for rows.Next() {
|
||||
item, err := scanEmployee(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, *item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
type ProjectPayload struct {
|
||||
Title string `json:"title"`
|
||||
DealType string `json:"deal_type"`
|
||||
OurPrice *float64 `json:"our_price"`
|
||||
Notes *string `json:"notes"`
|
||||
DLDPermit *string `json:"dld_permit"`
|
||||
Building *string `json:"building"`
|
||||
Bedrooms *int64 `json:"bedrooms"`
|
||||
SizeSqft *float64 `json:"size_sqft"`
|
||||
OurURL *string `json:"our_url"`
|
||||
}
|
||||
|
||||
func (a *App) Summary(ctx context.Context, emp *Employee) (map[string]any, error) {
|
||||
out := map[string]any{
|
||||
"projects_total": 0,
|
||||
"employees_total": 0,
|
||||
"listings_total": 0,
|
||||
"listings_active": 0,
|
||||
"listings_removed": 0,
|
||||
"scrape_interval_hours": a.Cfg.ScrapeIntervalHours,
|
||||
"bayut_enabled": false,
|
||||
}
|
||||
if emp == nil {
|
||||
return out, nil
|
||||
}
|
||||
var projects int64
|
||||
var listings, active, removed sql.NullInt64
|
||||
err := a.DB.QueryRowContext(ctx, `SELECT count(*) FROM projects WHERE owner_id = ?`, emp.ID).Scan(&projects)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = a.DB.QueryRowContext(ctx, `
|
||||
SELECT count(*),
|
||||
sum(CASE WHEN status IN ('ACTIVE','active') THEN 1 ELSE 0 END),
|
||||
sum(CASE WHEN status IN ('REMOVED','removed') THEN 1 ELSE 0 END)
|
||||
FROM competitor_listings l JOIN projects p ON p.id = l.project_id
|
||||
WHERE p.owner_id = ?`, emp.ID).Scan(&listings, &active, &removed)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out["projects_total"] = projects
|
||||
out["employees_total"] = 1
|
||||
out["listings_total"] = nullIntValue(listings)
|
||||
out["listings_active"] = nullIntValue(active)
|
||||
out["listings_removed"] = nullIntValue(removed)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (a *App) ListProjects(ctx context.Context, ownerID int64) ([]Project, error) {
|
||||
rows, err := a.DB.QueryContext(ctx, projectSelect()+` WHERE p.owner_id = ? ORDER BY p.created_at DESC`, ownerID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []Project{}
|
||||
for rows.Next() {
|
||||
p, err := a.scanProject(rows, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, *p)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (a *App) ProjectByID(ctx context.Context, ownerID, projectID int64, detail bool) (*Project, error) {
|
||||
row := a.DB.QueryRowContext(ctx, projectSelect()+` WHERE p.id = ? AND p.owner_id = ?`, projectID, ownerID)
|
||||
p, err := a.scanProject(row, detail)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (a *App) CreateProject(ctx context.Context, ownerID int64, p ProjectPayload) (*Project, error) {
|
||||
title := cleanString(p.Title)
|
||||
if title == "" {
|
||||
return nil, fmt.Errorf("title is required")
|
||||
}
|
||||
deal, err := enumDealIn(p.DealType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err := a.DB.ExecContext(ctx, `
|
||||
INSERT INTO projects
|
||||
(title, deal_type, owner_id, our_price, notes, dld_permit, building, bedrooms, size_sqft, our_url, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
title, deal, ownerID, p.OurPrice, cleanPtr(p.Notes), cleanPtr(p.DLDPermit), cleanPtr(p.Building),
|
||||
p.Bedrooms, p.SizeSqft, cleanPtr(p.OurURL), dbNow(),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
id, _ := res.LastInsertId()
|
||||
return a.ProjectByID(ctx, ownerID, id, true)
|
||||
}
|
||||
|
||||
func (a *App) UpdateProject(ctx context.Context, ownerID, projectID int64, p ProjectPayload) (*Project, error) {
|
||||
current, err := a.ProjectByID(ctx, ownerID, projectID, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
title := cleanString(p.Title)
|
||||
if title == "" {
|
||||
title = current.Title
|
||||
}
|
||||
deal := "SALE"
|
||||
if current.DealType == "rent" {
|
||||
deal = "RENT"
|
||||
}
|
||||
if p.DealType != "" {
|
||||
deal, err = enumDealIn(p.DealType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
_, err = a.DB.ExecContext(ctx, `
|
||||
UPDATE projects
|
||||
SET title = ?, deal_type = ?, our_price = ?, notes = ?, dld_permit = ?,
|
||||
building = ?, bedrooms = ?, size_sqft = ?, our_url = ?
|
||||
WHERE id = ? AND owner_id = ?`,
|
||||
title, deal, p.OurPrice, cleanPtr(p.Notes), cleanPtr(p.DLDPermit), cleanPtr(p.Building),
|
||||
p.Bedrooms, p.SizeSqft, cleanPtr(p.OurURL), projectID, ownerID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.ProjectByID(ctx, ownerID, projectID, true)
|
||||
}
|
||||
|
||||
func (a *App) DeleteProject(ctx context.Context, ownerID, projectID int64) error {
|
||||
tx, err := a.DB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
listingRows, err := tx.QueryContext(ctx, `SELECT id FROM competitor_listings WHERE project_id = ?`, projectID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
listingIDs := []int64{}
|
||||
for listingRows.Next() {
|
||||
var id int64
|
||||
if err := listingRows.Scan(&id); err != nil {
|
||||
listingRows.Close()
|
||||
return err
|
||||
}
|
||||
listingIDs = append(listingIDs, id)
|
||||
}
|
||||
listingRows.Close()
|
||||
for _, id := range listingIDs {
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM price_history WHERE listing_id = ?`, id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM competitor_listings WHERE project_id = ?`, projectID); err != nil {
|
||||
return err
|
||||
}
|
||||
res, err := tx.ExecContext(ctx, `DELETE FROM projects WHERE id = ? AND owner_id = ?`, projectID, ownerID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, _ := res.RowsAffected()
|
||||
if affected == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (a *App) DeleteListing(ctx context.Context, ownerID, listingID int64) error {
|
||||
var id int64
|
||||
err := a.DB.QueryRowContext(ctx, `
|
||||
SELECT l.id
|
||||
FROM competitor_listings l JOIN projects p ON p.id = l.project_id
|
||||
WHERE l.id = ? AND p.owner_id = ?`, listingID, ownerID).Scan(&id)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := a.DB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM price_history WHERE listing_id = ?`, id); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM competitor_listings WHERE id = ?`, id); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (a *App) ListingByID(ctx context.Context, id int64, withHistory bool) (*Listing, error) {
|
||||
row := a.DB.QueryRowContext(ctx, listingSelect()+` WHERE l.id = ?`, id)
|
||||
item, err := scanListing(row, withHistory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if withHistory {
|
||||
history, err := a.PriceHistory(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.PriceHistory = history
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func projectSelect() string {
|
||||
return `
|
||||
SELECT p.id, p.title, p.deal_type, p.our_price, p.notes, p.dld_permit, p.building,
|
||||
p.bedrooms, p.size_sqft, p.our_url, p.owner_id, p.created_at, p.last_checked_at,
|
||||
(SELECT count(*) FROM competitor_listings l WHERE l.project_id = p.id),
|
||||
(SELECT count(*) FROM competitor_listings l WHERE l.project_id = p.id AND l.status IN ('ACTIVE','active')),
|
||||
(SELECT count(*) FROM competitor_listings l WHERE l.project_id = p.id AND l.status IN ('REMOVED','removed')),
|
||||
(SELECT min(l.current_price) FROM competitor_listings l WHERE l.project_id = p.id AND l.status IN ('ACTIVE','active') AND l.current_price IS NOT NULL)
|
||||
FROM projects p`
|
||||
}
|
||||
|
||||
func (a *App) scanProject(row rowScanner, detail bool) (*Project, error) {
|
||||
var p Project
|
||||
var deal string
|
||||
var price, size, minPrice sql.NullFloat64
|
||||
var notes, permit, building, ourURL, created, checked sql.NullString
|
||||
var bedrooms sql.NullInt64
|
||||
if err := row.Scan(
|
||||
&p.ID, &p.Title, &deal, &price, ¬es, &permit, &building, &bedrooms, &size, &ourURL,
|
||||
&p.OwnerID, &created, &checked, &p.ListingsTotal, &p.ListingsActive, &p.ListingsRemoved, &minPrice,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.DealType = enumDealOut(deal)
|
||||
p.OurPrice = nullableFloat(price)
|
||||
p.Notes = nullableString(notes)
|
||||
p.DLDPermit = nullableString(permit)
|
||||
p.Building = nullableString(building)
|
||||
p.Bedrooms = nullableInt(bedrooms)
|
||||
p.SizeSqft = nullableFloat(size)
|
||||
p.OurURL = nullableString(ourURL)
|
||||
p.CreatedAt = timeOut(created)
|
||||
p.LastCheckedAt = timeOut(checked)
|
||||
p.MinCompetitorPrice = nullableFloat(minPrice)
|
||||
if owner, err := a.EmployeeByID(context.Background(), p.OwnerID); err == nil {
|
||||
p.Owner = owner
|
||||
}
|
||||
if detail {
|
||||
listings, err := a.ListingsForProject(context.Background(), p.ID, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.Listings = listings
|
||||
}
|
||||
return &p, nil
|
||||
}
|
||||
|
||||
func (a *App) ListingsForProject(ctx context.Context, projectID int64, withHistory bool) ([]Listing, error) {
|
||||
rows, err := a.DB.QueryContext(ctx, listingSelect()+` WHERE l.project_id = ? ORDER BY l.first_seen_at DESC`, projectID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []Listing{}
|
||||
for rows.Next() {
|
||||
item, err := scanListing(rows, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if withHistory {
|
||||
item.PriceHistory, err = a.PriceHistory(ctx, item.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
items = append(items, *item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (a *App) PriceHistory(ctx context.Context, listingID int64) ([]PricePoint, error) {
|
||||
rows, err := a.DB.QueryContext(ctx, `
|
||||
SELECT id, price, recorded_at
|
||||
FROM price_history
|
||||
WHERE listing_id = ?
|
||||
ORDER BY recorded_at DESC`, listingID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []PricePoint{}
|
||||
for rows.Next() {
|
||||
var p PricePoint
|
||||
var price sql.NullFloat64
|
||||
var recorded sql.NullString
|
||||
if err := rows.Scan(&p.ID, &price, &recorded); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.Price = nullableFloat(price)
|
||||
p.RecordedAt = timeOut(recorded)
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func listingSelect() string {
|
||||
return `
|
||||
SELECT l.id, l.project_id, l.source, l.external_id, l.url, l.title, l.agent_name,
|
||||
l.agency_name, l.current_price, l.currency, l.status, l.first_seen_at, l.last_seen_at
|
||||
FROM competitor_listings l`
|
||||
}
|
||||
|
||||
func scanListing(row rowScanner, _ bool) (*Listing, error) {
|
||||
var l Listing
|
||||
var source, status string
|
||||
var title, agent, agency, currency, firstSeen, lastSeen sql.NullString
|
||||
var price sql.NullFloat64
|
||||
if err := row.Scan(
|
||||
&l.ID, &l.ProjectID, &source, &l.ExternalID, &l.URL, &title, &agent, &agency,
|
||||
&price, ¤cy, &status, &firstSeen, &lastSeen,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l.Source = enumSourceOut(source)
|
||||
l.Title = nullableString(title)
|
||||
l.AgentName = nullableString(agent)
|
||||
l.AgencyName = nullableString(agency)
|
||||
l.CurrentPrice = nullableFloat(price)
|
||||
l.Currency = nullableString(currency)
|
||||
l.Status = enumStatusOut(status)
|
||||
l.FirstSeenAt = timeOut(firstSeen)
|
||||
l.LastSeenAt = timeOut(lastSeen)
|
||||
return &l, nil
|
||||
}
|
||||
|
||||
func nullIfEmpty(value string) *string {
|
||||
value = cleanString(value)
|
||||
if value == "" {
|
||||
return nil
|
||||
}
|
||||
return &value
|
||||
}
|
||||
|
||||
func nullIntValue(value sql.NullInt64) int64 {
|
||||
if !value.Valid {
|
||||
return 0
|
||||
}
|
||||
return value.Int64
|
||||
}
|
||||
129
internal/pf/telegram.go
Normal file
129
internal/pf/telegram.go
Normal file
@@ -0,0 +1,129 @@
|
||||
package pf
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Telegram struct {
|
||||
token string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
type tgResponse[T any] struct {
|
||||
OK bool `json:"ok"`
|
||||
Description string `json:"description"`
|
||||
Result T `json:"result"`
|
||||
}
|
||||
|
||||
type TGUpdate struct {
|
||||
UpdateID int64 `json:"update_id"`
|
||||
Message *TGMessage `json:"message"`
|
||||
}
|
||||
|
||||
type TGMessage struct {
|
||||
MessageID int64 `json:"message_id"`
|
||||
Text string `json:"text"`
|
||||
Chat TGChat `json:"chat"`
|
||||
From *TGUser `json:"from"`
|
||||
}
|
||||
|
||||
type TGChat struct {
|
||||
ID int64 `json:"id"`
|
||||
}
|
||||
|
||||
type TGUser struct {
|
||||
ID int64 `json:"id"`
|
||||
Username string `json:"username"`
|
||||
FirstName string `json:"first_name"`
|
||||
LastName string `json:"last_name"`
|
||||
}
|
||||
|
||||
func NewTelegram(token string) *Telegram {
|
||||
return &Telegram{token: token, client: &http.Client{Timeout: 35 * time.Second}}
|
||||
}
|
||||
|
||||
func (t *Telegram) Enabled() bool {
|
||||
return strings.TrimSpace(t.token) != ""
|
||||
}
|
||||
|
||||
func (t *Telegram) SendMessage(ctx context.Context, chatID string, text string) error {
|
||||
if !t.Enabled() {
|
||||
return nil
|
||||
}
|
||||
payload := map[string]any{
|
||||
"chat_id": chatID,
|
||||
"text": text,
|
||||
"parse_mode": "HTML",
|
||||
"disable_web_page_preview": false,
|
||||
}
|
||||
body, _ := json.Marshal(payload)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, t.apiURL("sendMessage"), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := t.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var out tgResponse[json.RawMessage]
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return err
|
||||
}
|
||||
if !out.OK {
|
||||
return errors.New(out.Description)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Telegram) GetUpdates(ctx context.Context, offset int64) ([]TGUpdate, error) {
|
||||
if !t.Enabled() {
|
||||
return nil, fmt.Errorf("TG_BOT_TOKEN не задан в k8s/secrets.yaml")
|
||||
}
|
||||
values := url.Values{}
|
||||
values.Set("timeout", "25")
|
||||
if offset > 0 {
|
||||
values.Set("offset", fmt.Sprintf("%d", offset))
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, t.apiURL("getUpdates")+"?"+values.Encode(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := t.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var out tgResponse[[]TGUpdate]
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !out.OK {
|
||||
return nil, errors.New(out.Description)
|
||||
}
|
||||
return out.Result, nil
|
||||
}
|
||||
|
||||
func (t *Telegram) apiURL(method string) string {
|
||||
return "https://api.telegram.org/bot" + t.token + "/" + method
|
||||
}
|
||||
|
||||
func (u TGUser) FullName() string {
|
||||
name := strings.TrimSpace(strings.TrimSpace(u.FirstName) + " " + strings.TrimSpace(u.LastName))
|
||||
if name != "" {
|
||||
return name
|
||||
}
|
||||
if u.Username != "" {
|
||||
return u.Username
|
||||
}
|
||||
return fmt.Sprintf("user_%d", u.ID)
|
||||
}
|
||||
134
internal/pf/worker.go
Normal file
134
internal/pf/worker.go
Normal file
@@ -0,0 +1,134 @@
|
||||
package pf
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Worker struct {
|
||||
python string
|
||||
module string
|
||||
}
|
||||
|
||||
type workerError struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
type workerListing struct {
|
||||
ListingID int64 `json:"listing_id"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
type BulkResult struct {
|
||||
Added int `json:"added"`
|
||||
Skipped int `json:"skipped"`
|
||||
Errors []string `json:"errors"`
|
||||
}
|
||||
|
||||
type CheckResult struct {
|
||||
Changes int `json:"changes"`
|
||||
}
|
||||
|
||||
type Suggestion struct {
|
||||
Source string `json:"source"`
|
||||
ExternalID string `json:"external_id"`
|
||||
URL string `json:"url"`
|
||||
Title *string `json:"title"`
|
||||
Price *float64 `json:"price"`
|
||||
Currency *string `json:"currency"`
|
||||
PermitNumber *string `json:"permit_number"`
|
||||
AgentName *string `json:"agent_name"`
|
||||
AgencyName *string `json:"agency_name"`
|
||||
IsActive bool `json:"is_active"`
|
||||
}
|
||||
|
||||
type SuggestionsResponse struct {
|
||||
OurPermit *string `json:"our_permit"`
|
||||
BayutEnabled bool `json:"bayut_enabled"`
|
||||
Suggestions struct {
|
||||
PropertyFinder []Suggestion `json:"propertyfinder"`
|
||||
Bayut []Suggestion `json:"bayut"`
|
||||
} `json:"suggestions"`
|
||||
}
|
||||
|
||||
func NewWorker(cfg Config) *Worker {
|
||||
return &Worker{python: cfg.WorkerPython, module: cfg.WorkerModule}
|
||||
}
|
||||
|
||||
func (w *Worker) AddListing(ctx context.Context, projectID int64, url string) (int64, error) {
|
||||
var out workerListing
|
||||
err := w.call(ctx, "add-listing", map[string]any{"project_id": projectID, "url": url}, &out)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if out.Error != "" {
|
||||
return 0, errors.New(out.Error)
|
||||
}
|
||||
return out.ListingID, nil
|
||||
}
|
||||
|
||||
func (w *Worker) AddListings(ctx context.Context, projectID int64, urls []string) (*BulkResult, error) {
|
||||
var out BulkResult
|
||||
if err := w.call(ctx, "add-listings", map[string]any{"project_id": projectID, "urls": urls}, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
func (w *Worker) CheckProject(ctx context.Context, projectID int64) (int, error) {
|
||||
var out CheckResult
|
||||
if err := w.call(ctx, "check-project", map[string]any{"project_id": projectID}, &out); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return out.Changes, nil
|
||||
}
|
||||
|
||||
func (w *Worker) CheckAll(ctx context.Context) (map[string]int, error) {
|
||||
var out map[string]int
|
||||
if err := w.call(ctx, "check-all", map[string]any{}, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (w *Worker) Suggest(ctx context.Context, projectID int64) (*SuggestionsResponse, error) {
|
||||
var out SuggestionsResponse
|
||||
if err := w.call(ctx, "suggest", map[string]any{"project_id": projectID}, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
func (w *Worker) call(ctx context.Context, command string, payload any, out any) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, w.python, "-m", w.module, command)
|
||||
cmd.Stdin = bytes.NewReader(body)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
var apiErr workerError
|
||||
if json.Unmarshal(stdout.Bytes(), &apiErr) == nil && apiErr.Error != "" {
|
||||
return errors.New(apiErr.Error)
|
||||
}
|
||||
if stderr.Len() > 0 {
|
||||
return errors.New(stderr.String())
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(stdout.Bytes(), out); err != nil {
|
||||
return fmt.Errorf("worker json decode failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user