6b762068fd
Results: - XGBoost (Optuna 100 trials): AUC=0.7856, Precision@3=0.5783 - LightGBM (Optuna 100 trials): AUC=0.7833, Precision@3=0.5736 - MLP (3 layers 256-128-64): AUC=0.7743, Precision@3=0.5643 - Ensemble (weighted voting): AUC=0.7840, Precision@3=0.5814 Baseline XGBoost: Precision@3=0.5287 Delta: +0.0527 (+5.3%) — DEPLOY threshold met (+5%) Latency: 35ms/race, 69ms/full-day (well under 200ms limit) SHAP: 31/43 features selected, top features: rang_cote, implied_prob, cote_direct, ratio_cote_field All 12 regression/latency tests passing. Co-Authored-By: Paperclip <noreply@paperclip.ing>
449 lines
16 KiB
Python
449 lines
16 KiB
Python
"""
|
|
Beta Monitoring — SaaS Turf Prédictions IA
|
|
Sprint 8 — QA, Beta Fermee, Go/No-Go
|
|
Ticket: HRT-34
|
|
|
|
Ce module :
|
|
- Collecte les feedbacks beta via l'API in-app
|
|
- Envoie des alertes Telegram en cas d'erreur détectée pendant la beta
|
|
- Génère le rapport beta final (bugs, UX, NPS)
|
|
|
|
Usage :
|
|
# Démarrer le monitoring beta
|
|
python tests/beta_monitor.py --watch --interval 60
|
|
|
|
# Générer le rapport beta final
|
|
python tests/beta_monitor.py --report
|
|
|
|
# Test d'envoi Telegram
|
|
python tests/beta_monitor.py --test-telegram
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import time
|
|
import sqlite3
|
|
import requests
|
|
import argparse
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
# ============================================================
|
|
# Configuration
|
|
# ============================================================
|
|
|
|
BASE_URL = os.environ.get("APP_URL", "http://localhost:8792")
|
|
TELEGRAM_TOKEN = os.environ.get(
|
|
"TELEGRAM_TOKEN", "8649773134:AAFqzZVtSHfPPFDadcte1B-1h23nZ8DmdYE"
|
|
)
|
|
TELEGRAM_CHAT_ID = os.environ.get("TELEGRAM_CHAT_ID", "") # À configurer
|
|
|
|
BETA_DB_PATH = os.environ.get("BETA_DB_PATH", "/home/h3r7/turf_saas/turf_saas.db")
|
|
REPORTS_DIR = Path("tests/reports")
|
|
REPORTS_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Seuils d'alerte
|
|
ERROR_RATE_THRESHOLD = 0.01 # 1% d'erreurs → alerte
|
|
LATENCY_P95_THRESHOLD_MS = 500 # p95 > 500ms → alerte
|
|
BETA_MIN_USERS = 10 # Minimum d'utilisateurs beta requis
|
|
NPS_TARGET = 7.0 # NPS cible (sur 10)
|
|
|
|
|
|
# ============================================================
|
|
# Alertes Telegram
|
|
# ============================================================
|
|
|
|
|
|
def send_telegram(message: str, parse_mode: str = "Markdown") -> bool:
|
|
"""Envoie un message Telegram d'alerte."""
|
|
if not TELEGRAM_TOKEN or not TELEGRAM_CHAT_ID:
|
|
print(f"⚠️ Telegram non configuré. Message: {message[:100]}")
|
|
return False
|
|
|
|
try:
|
|
resp = requests.post(
|
|
f"https://api.telegram.org/bot{TELEGRAM_TOKEN}/sendMessage",
|
|
json={
|
|
"chat_id": TELEGRAM_CHAT_ID,
|
|
"text": message,
|
|
"parse_mode": parse_mode,
|
|
},
|
|
timeout=10,
|
|
)
|
|
if resp.status_code == 200:
|
|
print(f"✅ Alerte Telegram envoyée")
|
|
return True
|
|
else:
|
|
print(f"❌ Telegram erreur: {resp.status_code} — {resp.text}")
|
|
return False
|
|
except Exception as e:
|
|
print(f"❌ Telegram exception: {e}")
|
|
return False
|
|
|
|
|
|
def alert_error(endpoint: str, status_code: int, message: str):
|
|
"""Alerte Telegram sur erreur critique."""
|
|
text = (
|
|
f"🚨 *ALERTE BETA — SaaS Turf IA*\n\n"
|
|
f"Erreur détectée sur `{endpoint}`\n"
|
|
f"Status: `{status_code}`\n"
|
|
f"Message: {message[:200]}\n"
|
|
f"Heure: {datetime.now().strftime('%H:%M:%S')}\n\n"
|
|
f"_Ticket: HRT-34_"
|
|
)
|
|
send_telegram(text)
|
|
|
|
|
|
def alert_performance(p95_ms: float, error_rate: float):
|
|
"""Alerte Telegram sur dégradation de performance."""
|
|
text = (
|
|
f"⚠️ *ALERTE PERFORMANCE — SaaS Turf IA*\n\n"
|
|
f"p95 latence: `{p95_ms:.0f}ms` (seuil: {LATENCY_P95_THRESHOLD_MS}ms)\n"
|
|
f"Error rate: `{error_rate * 100:.2f}%` (seuil: {ERROR_RATE_THRESHOLD * 100:.1f}%)\n"
|
|
f"Heure: {datetime.now().strftime('%H:%M:%S')}\n\n"
|
|
f"_Ticket: HRT-34_"
|
|
)
|
|
send_telegram(text)
|
|
|
|
|
|
# ============================================================
|
|
# Collecte de métriques
|
|
# ============================================================
|
|
|
|
|
|
class BetaMonitor:
|
|
"""Moniteur actif pendant la beta fermée."""
|
|
|
|
ENDPOINTS_TO_CHECK = [
|
|
"/api",
|
|
"/api/races",
|
|
"/api/scoring",
|
|
"/dashboard",
|
|
"/",
|
|
]
|
|
|
|
def __init__(self, base_url: str = BASE_URL):
|
|
self.base_url = base_url.rstrip("/")
|
|
self.errors: list[dict] = []
|
|
self.latencies: list[float] = []
|
|
self.check_count = 0
|
|
|
|
def check_endpoint(self, path: str) -> dict:
|
|
"""Vérifie un endpoint et retourne le résultat."""
|
|
start = time.time()
|
|
try:
|
|
resp = requests.get(f"{self.base_url}{path}", timeout=10)
|
|
latency_ms = (time.time() - start) * 1000
|
|
return {
|
|
"path": path,
|
|
"status": resp.status_code,
|
|
"latency_ms": latency_ms,
|
|
"ok": resp.status_code < 500,
|
|
"timestamp": datetime.now().isoformat(),
|
|
}
|
|
except requests.exceptions.ConnectionError as e:
|
|
return {
|
|
"path": path,
|
|
"status": 0,
|
|
"latency_ms": 0,
|
|
"ok": False,
|
|
"error": str(e),
|
|
"timestamp": datetime.now().isoformat(),
|
|
}
|
|
except Exception as e:
|
|
return {
|
|
"path": path,
|
|
"status": 0,
|
|
"latency_ms": 0,
|
|
"ok": False,
|
|
"error": str(e),
|
|
"timestamp": datetime.now().isoformat(),
|
|
}
|
|
|
|
def run_checks(self) -> dict:
|
|
"""Exécute tous les checks et retourne un résumé."""
|
|
results = [self.check_endpoint(p) for p in self.ENDPOINTS_TO_CHECK]
|
|
self.check_count += 1
|
|
|
|
failures = [r for r in results if not r["ok"]]
|
|
latencies = [r["latency_ms"] for r in results if r["latency_ms"] > 0]
|
|
|
|
p95 = (
|
|
sorted(latencies)[int(len(latencies) * 0.95)]
|
|
if len(latencies) >= 2
|
|
else (latencies[0] if latencies else 0)
|
|
)
|
|
error_rate = len(failures) / len(results) if results else 0
|
|
|
|
# Stocker pour rapport
|
|
self.latencies.extend(latencies)
|
|
self.errors.extend(failures)
|
|
|
|
return {
|
|
"check_number": self.check_count,
|
|
"timestamp": datetime.now().isoformat(),
|
|
"total_checks": len(results),
|
|
"failures": len(failures),
|
|
"error_rate": error_rate,
|
|
"p95_ms": p95,
|
|
"results": results,
|
|
}
|
|
|
|
def watch(self, interval_seconds: int = 60):
|
|
"""Surveillance continue avec alertes Telegram."""
|
|
print(f"🔍 Beta monitoring démarré — {self.base_url}")
|
|
print(f" Intervalle: {interval_seconds}s")
|
|
print(f" Endpoints: {len(self.ENDPOINTS_TO_CHECK)}")
|
|
print(f" Ctrl+C pour arrêter\n")
|
|
|
|
consecutive_errors = 0
|
|
|
|
try:
|
|
while True:
|
|
summary = self.run_checks()
|
|
timestamp = datetime.now().strftime("%H:%M:%S")
|
|
|
|
status_icon = "✅" if summary["error_rate"] == 0 else "❌"
|
|
print(
|
|
f"[{timestamp}] {status_icon} "
|
|
f"Check #{summary['check_number']} — "
|
|
f"p95={summary['p95_ms']:.0f}ms, "
|
|
f"errors={summary['failures']}/{summary['total_checks']}"
|
|
)
|
|
|
|
# Alertes
|
|
if summary["error_rate"] > ERROR_RATE_THRESHOLD:
|
|
consecutive_errors += 1
|
|
if consecutive_errors >= 2: # 2 checks consécutifs en erreur
|
|
for failure in summary["results"]:
|
|
if not failure["ok"]:
|
|
alert_error(
|
|
failure["path"],
|
|
failure.get("status", 0),
|
|
failure.get("error", "Non-2xx response"),
|
|
)
|
|
else:
|
|
consecutive_errors = 0
|
|
|
|
if summary["p95_ms"] > LATENCY_P95_THRESHOLD_MS:
|
|
print(f"⚠️ Latence p95 élevée: {summary['p95_ms']:.0f}ms")
|
|
if summary["p95_ms"] > LATENCY_P95_THRESHOLD_MS * 2:
|
|
alert_performance(summary["p95_ms"], summary["error_rate"])
|
|
|
|
# Sauvegarder les résultats
|
|
log_file = REPORTS_DIR / "beta_monitor_log.jsonl"
|
|
with open(log_file, "a") as f:
|
|
f.write(json.dumps(summary) + "\n")
|
|
|
|
time.sleep(interval_seconds)
|
|
|
|
except KeyboardInterrupt:
|
|
print(f"\n⏹️ Monitoring arrêté après {self.check_count} checks")
|
|
self.generate_report()
|
|
|
|
|
|
# ============================================================
|
|
# Rapport beta final
|
|
# ============================================================
|
|
|
|
|
|
class BetaReport:
|
|
"""Générateur de rapport beta fermée."""
|
|
|
|
def __init__(self, base_url: str = BASE_URL):
|
|
self.base_url = base_url
|
|
self.timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
|
|
def collect_feedback_from_db(self) -> list[dict]:
|
|
"""Collecte les feedbacks depuis la BDD (table beta_feedback si elle existe)."""
|
|
try:
|
|
conn = sqlite3.connect(BETA_DB_PATH)
|
|
c = conn.cursor()
|
|
c.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='table' AND name='beta_feedback'"
|
|
)
|
|
if not c.fetchone():
|
|
conn.close()
|
|
return []
|
|
c.execute("SELECT * FROM beta_feedback ORDER BY created_at DESC")
|
|
rows = c.fetchall()
|
|
conn.close()
|
|
return [dict(zip([col[0] for col in c.description], row)) for row in rows]
|
|
except Exception as e:
|
|
print(f"⚠️ Impossible de lire beta_feedback: {e}")
|
|
return []
|
|
|
|
def collect_monitor_logs(self) -> list[dict]:
|
|
"""Lit les logs du monitoring beta."""
|
|
log_file = REPORTS_DIR / "beta_monitor_log.jsonl"
|
|
if not log_file.exists():
|
|
return []
|
|
entries = []
|
|
with open(log_file) as f:
|
|
for line in f:
|
|
try:
|
|
entries.append(json.loads(line))
|
|
except Exception:
|
|
pass
|
|
return entries
|
|
|
|
def generate(self) -> str:
|
|
"""Génère le rapport complet et le sauvegarde."""
|
|
feedbacks = self.collect_feedback_from_db()
|
|
monitor_logs = self.collect_monitor_logs()
|
|
|
|
# Calculer NPS depuis les feedbacks
|
|
nps_scores = [
|
|
f.get("nps_score") for f in feedbacks if f.get("nps_score") is not None
|
|
]
|
|
avg_nps = sum(nps_scores) / len(nps_scores) if nps_scores else None
|
|
|
|
# Statistiques monitoring
|
|
if monitor_logs:
|
|
all_latencies = []
|
|
total_errors = 0
|
|
total_checks = 0
|
|
for entry in monitor_logs:
|
|
all_latencies.extend(
|
|
[
|
|
r["latency_ms"]
|
|
for r in entry.get("results", [])
|
|
if r.get("latency_ms", 0) > 0
|
|
]
|
|
)
|
|
total_errors += entry.get("failures", 0)
|
|
total_checks += entry.get("total_checks", 0)
|
|
avg_latency = (
|
|
sum(all_latencies) / len(all_latencies) if all_latencies else 0
|
|
)
|
|
overall_error_rate = total_errors / total_checks if total_checks > 0 else 0
|
|
else:
|
|
avg_latency = 0
|
|
overall_error_rate = 0
|
|
total_checks = 0
|
|
|
|
# Construire le rapport
|
|
report = []
|
|
report.append("=" * 60)
|
|
report.append("RAPPORT BETA FERMÉE — SaaS Turf Prédictions IA")
|
|
report.append(f"Généré le : {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
|
report.append(f"Ticket : HRT-34")
|
|
report.append("=" * 60)
|
|
report.append("")
|
|
report.append("## 1. PARTICIPANTS BETA")
|
|
report.append(f" Feedbacks reçus : {len(feedbacks)}")
|
|
report.append(
|
|
f" NPS moyen : {avg_nps:.1f}/10"
|
|
if avg_nps
|
|
else " NPS moyen : (en attente feedbacks)"
|
|
)
|
|
report.append(f" Cible NPS : ≥ {NPS_TARGET}/10")
|
|
nps_ok = avg_nps is not None and avg_nps >= NPS_TARGET
|
|
report.append(
|
|
f" Statut NPS : {'✅ OBJECTIF ATTEINT' if nps_ok else '⏳ En attente' if avg_nps is None else '❌ OBJECTIF NON ATTEINT'}"
|
|
)
|
|
report.append("")
|
|
report.append("## 2. BUGS SIGNALÉS")
|
|
bugs = [f for f in feedbacks if f.get("type") == "bug"]
|
|
critical_bugs = [b for b in bugs if b.get("severity") in ("critical", "high")]
|
|
report.append(f" Total bugs : {len(bugs)}")
|
|
report.append(f" Critiques/High : {len(critical_bugs)}")
|
|
report.append(
|
|
f" Statut : {'✅ 0 bug critique' if len(critical_bugs) == 0 else f'❌ {len(critical_bugs)} bug(s) critique(s)'}"
|
|
)
|
|
report.append("")
|
|
report.append("## 3. PERFORMANCE RÉELLE (monitoring)")
|
|
report.append(f" Checks effectués: {total_checks}")
|
|
report.append(f" Latence moyenne : {avg_latency:.1f}ms")
|
|
report.append(f" Error rate : {overall_error_rate * 100:.2f}%")
|
|
report.append(f" Seuil latence : {LATENCY_P95_THRESHOLD_MS}ms")
|
|
perf_ok = (
|
|
avg_latency < LATENCY_P95_THRESHOLD_MS
|
|
and overall_error_rate < ERROR_RATE_THRESHOLD
|
|
)
|
|
report.append(
|
|
f" Statut : {'✅ OBJECTIF ATTEINT' if perf_ok else '⏳ Données insuffisantes' if total_checks == 0 else '❌ OBJECTIF NON ATTEINT'}"
|
|
)
|
|
report.append("")
|
|
report.append("## 4. FEEDBACKS UX")
|
|
ux_feedbacks = [f for f in feedbacks if f.get("type") == "ux"]
|
|
report.append(f" Retours UX : {len(ux_feedbacks)}")
|
|
if ux_feedbacks:
|
|
for fb in ux_feedbacks[:5]: # Top 5
|
|
report.append(f" - {fb.get('comment', '')[:100]}")
|
|
report.append("")
|
|
report.append("## 5. VERDICT BETA FERMÉE")
|
|
users_ok = len(feedbacks) >= 5 # Au moins 5 feedbacks = 5 users satisfaits
|
|
verdict = all([users_ok, nps_ok, len(critical_bugs) == 0])
|
|
report.append(
|
|
f" Participants suffisants (≥5) : {'✅' if users_ok else '❌'}"
|
|
)
|
|
report.append(f" NPS ≥ 7/10 : {'✅' if nps_ok else '❌'}")
|
|
report.append(
|
|
f" 0 bug critique : {'✅' if len(critical_bugs) == 0 else '❌'}"
|
|
)
|
|
report.append("")
|
|
report.append(
|
|
f" VERDICT GLOBAL : {'✅ GO — Beta réussie' if verdict else '❌ NO-GO — Conditions non remplies'}"
|
|
)
|
|
report.append("=" * 60)
|
|
|
|
report_text = "\n".join(report)
|
|
|
|
# Sauvegarder
|
|
report_file = REPORTS_DIR / f"beta_report_{self.timestamp}.txt"
|
|
with open(report_file, "w") as f:
|
|
f.write(report_text)
|
|
|
|
print(report_text)
|
|
print(f"\nRapport sauvegardé : {report_file}")
|
|
|
|
return report_text
|
|
|
|
|
|
# ============================================================
|
|
# CLI
|
|
# ============================================================
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Beta Monitor — SaaS Turf IA")
|
|
parser.add_argument("--watch", action="store_true", help="Surveillance continue")
|
|
parser.add_argument(
|
|
"--interval", type=int, default=60, help="Intervalle en secondes (défaut: 60)"
|
|
)
|
|
parser.add_argument(
|
|
"--report", action="store_true", help="Générer le rapport beta final"
|
|
)
|
|
parser.add_argument(
|
|
"--test-telegram", action="store_true", help="Tester l'envoi Telegram"
|
|
)
|
|
parser.add_argument(
|
|
"--url", default=BASE_URL, help=f"URL de l'app (défaut: {BASE_URL})"
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
if args.test_telegram:
|
|
print("Test d'envoi Telegram...")
|
|
ok = send_telegram(
|
|
"✅ *Test alerte Beta* — SaaS Turf IA\n_Ceci est un test du système d'alertes QA_\nTicket: HRT-34"
|
|
)
|
|
sys.exit(0 if ok else 1)
|
|
|
|
if args.report:
|
|
reporter = BetaReport(args.url)
|
|
reporter.generate()
|
|
sys.exit(0)
|
|
|
|
if args.watch:
|
|
monitor = BetaMonitor(args.url)
|
|
monitor.watch(interval_seconds=args.interval)
|
|
sys.exit(0)
|
|
|
|
parser.print_help()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|