refactor: reorganize project structure by moving core modules and update import paths in API server
This commit is contained in:
@@ -0,0 +1,361 @@
|
||||
"""
|
||||
Model Manager - Hệ thống quản lý và vận hành tất cả các loại models
|
||||
Hỗ trợ: XGBoost, Random Forest, Decision Tree, SVM, CNN, và các model khác
|
||||
"""
|
||||
|
||||
import joblib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, List, Any, Tuple
|
||||
from datetime import datetime
|
||||
import numpy as np
|
||||
import warnings
|
||||
|
||||
# PyTorch for CNN models
|
||||
try:
|
||||
import torch
|
||||
PYTORCH_AVAILABLE = True
|
||||
except ImportError:
|
||||
PYTORCH_AVAILABLE = False
|
||||
|
||||
warnings.filterwarnings('ignore')
|
||||
|
||||
|
||||
class ModelManager:
|
||||
"""Quản lý tất cả các models: load, save, list, validate"""
|
||||
|
||||
def __init__(self, models_dir: str = "model_train"):
|
||||
self.models_dir = Path(models_dir)
|
||||
self.models_dir.mkdir(exist_ok=True)
|
||||
self.current_model = None
|
||||
self.current_metadata = None
|
||||
|
||||
def list_models(self) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Liệt kê tất cả models có sẵn với metadata
|
||||
|
||||
Returns:
|
||||
List of dicts containing model info
|
||||
"""
|
||||
models = []
|
||||
|
||||
# Tìm tất cả file .joblib
|
||||
for model_file in self.models_dir.glob("*.joblib"):
|
||||
# Skip Zone.Identifier files
|
||||
if "Zone.Identifier" in model_file.name:
|
||||
continue
|
||||
|
||||
model_info = {
|
||||
"filename": model_file.name,
|
||||
"path": str(model_file),
|
||||
"size_mb": model_file.stat().st_size / (1024 * 1024),
|
||||
"modified": datetime.fromtimestamp(model_file.stat().st_mtime).isoformat(),
|
||||
}
|
||||
|
||||
# Tìm metadata file tương ứng
|
||||
metadata_file = model_file.with_suffix('.json')
|
||||
if not metadata_file.exists():
|
||||
# Try with _info.json suffix
|
||||
metadata_file = model_file.parent / (model_file.stem + "_info.json")
|
||||
|
||||
if metadata_file.exists():
|
||||
try:
|
||||
with open(metadata_file, 'r') as f:
|
||||
metadata = json.load(f)
|
||||
model_info["metadata"] = metadata
|
||||
model_info["has_metadata"] = True
|
||||
|
||||
# Extract key info
|
||||
model_info["model_type"] = metadata.get("model_type", "unknown")
|
||||
model_info["features"] = metadata.get("features", [])
|
||||
model_info["n_features"] = metadata.get("n_features", 0)
|
||||
model_info["n_classes"] = metadata.get("n_classes", 0)
|
||||
model_info["test_accuracy"] = metadata.get("test_accuracy", None)
|
||||
model_info["timestamp"] = metadata.get("timestamp", None)
|
||||
model_info["data_source"] = metadata.get("data_source", "unknown")
|
||||
|
||||
except Exception as e:
|
||||
model_info["has_metadata"] = False
|
||||
model_info["metadata_error"] = str(e)
|
||||
else:
|
||||
model_info["has_metadata"] = False
|
||||
|
||||
models.append(model_info)
|
||||
|
||||
# Sort by modified time (newest first)
|
||||
models.sort(key=lambda x: x["modified"], reverse=True)
|
||||
|
||||
return models
|
||||
|
||||
def load_model(self, model_filename: str) -> Tuple[Any, Optional[Any], Dict[str, Any]]:
|
||||
"""
|
||||
Load model từ file
|
||||
|
||||
Args:
|
||||
model_filename: Tên file model (ví dụ: "model_odc.joblib")
|
||||
|
||||
Returns:
|
||||
Tuple of (model, label_encoder, metadata)
|
||||
"""
|
||||
model_path = self.models_dir / model_filename
|
||||
|
||||
if not model_path.exists():
|
||||
raise FileNotFoundError(f"Model không tồn tại: {model_filename}")
|
||||
|
||||
# Load model
|
||||
print(f"[MODEL MANAGER] Loading model: {model_filename}")
|
||||
model_data = joblib.load(model_path)
|
||||
|
||||
# Extract model and encoder
|
||||
if isinstance(model_data, dict):
|
||||
model = model_data.get('model')
|
||||
label_encoder = model_data.get('label_encoder')
|
||||
else:
|
||||
# Old format: model only
|
||||
model = model_data
|
||||
label_encoder = None
|
||||
|
||||
# Load metadata
|
||||
metadata = self._load_metadata(model_filename)
|
||||
|
||||
# Store current model
|
||||
self.current_model = model
|
||||
self.current_metadata = metadata
|
||||
|
||||
# Check if CNN model and set to eval mode
|
||||
if PYTORCH_AVAILABLE and hasattr(model, '__class__') and 'CNN' in model.__class__.__name__:
|
||||
model.eval()
|
||||
print(f"[MODEL MANAGER] PyTorch CNN model detected and set to eval mode")
|
||||
|
||||
print(f"[MODEL MANAGER] Model loaded successfully")
|
||||
print(f" - Type: {metadata.get('model_type', 'unknown')}")
|
||||
print(f" - Features: {metadata.get('n_features', 'N/A')}")
|
||||
print(f" - Classes: {metadata.get('n_classes', 'N/A')}")
|
||||
print(f" - Accuracy: {metadata.get('test_accuracy', 'N/A')}")
|
||||
|
||||
return model, label_encoder, metadata
|
||||
|
||||
def _load_metadata(self, model_filename: str) -> Dict[str, Any]:
|
||||
"""Load metadata cho model"""
|
||||
model_path = self.models_dir / model_filename
|
||||
|
||||
# Try multiple metadata file patterns
|
||||
metadata_files = [
|
||||
model_path.with_suffix('.json'),
|
||||
model_path.parent / (model_path.stem + "_info.json"),
|
||||
]
|
||||
|
||||
for metadata_file in metadata_files:
|
||||
if metadata_file.exists():
|
||||
try:
|
||||
with open(metadata_file, 'r') as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
print(f"[MODEL MANAGER] Warning: Could not load metadata from {metadata_file}: {e}")
|
||||
|
||||
# Return default metadata if not found
|
||||
print(f"[MODEL MANAGER] Warning: No metadata found for {model_filename}")
|
||||
return {
|
||||
"model_type": "unknown",
|
||||
"features": [],
|
||||
"n_features": 0,
|
||||
"n_classes": 0,
|
||||
"timestamp": None
|
||||
}
|
||||
|
||||
def save_model(self, model: Any, metadata: Dict[str, Any],
|
||||
model_filename: Optional[str] = None,
|
||||
label_encoder: Optional[Any] = None) -> str:
|
||||
"""
|
||||
Save model với metadata
|
||||
|
||||
Args:
|
||||
model: Model object
|
||||
metadata: Dict chứa thông tin về model
|
||||
model_filename: Tên file (optional, sẽ auto-generate nếu không có)
|
||||
label_encoder: Label encoder (optional)
|
||||
|
||||
Returns:
|
||||
Path to saved model file
|
||||
"""
|
||||
# Generate filename if not provided
|
||||
if model_filename is None:
|
||||
model_type = metadata.get("model_type", "model")
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
model_filename = f"model_{model_type}_{timestamp}.joblib"
|
||||
|
||||
model_path = self.models_dir / model_filename
|
||||
metadata_path = model_path.parent / (model_path.stem + "_info.json")
|
||||
|
||||
# Prepare model data
|
||||
if label_encoder is not None:
|
||||
model_data = {
|
||||
'model': model,
|
||||
'label_encoder': label_encoder
|
||||
}
|
||||
else:
|
||||
model_data = {
|
||||
'model': model
|
||||
}
|
||||
|
||||
# Save model
|
||||
print(f"[MODEL MANAGER] Saving model to: {model_path}")
|
||||
joblib.dump(model_data, model_path)
|
||||
|
||||
# Save metadata
|
||||
print(f"[MODEL MANAGER] Saving metadata to: {metadata_path}")
|
||||
with open(metadata_path, 'w') as f:
|
||||
json.dump(metadata, f, indent=2)
|
||||
|
||||
print(f"[MODEL MANAGER] Model saved successfully!")
|
||||
|
||||
return str(model_path)
|
||||
|
||||
def validate_model(self, model_filename: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Validate model file và kiểm tra integrity
|
||||
|
||||
Returns:
|
||||
Dict with validation results
|
||||
"""
|
||||
result = {
|
||||
"valid": False,
|
||||
"errors": [],
|
||||
"warnings": []
|
||||
}
|
||||
|
||||
model_path = self.models_dir / model_filename
|
||||
|
||||
# Check file exists
|
||||
if not model_path.exists():
|
||||
result["errors"].append(f"File không tồn tại: {model_filename}")
|
||||
return result
|
||||
|
||||
# Try to load model
|
||||
try:
|
||||
model, encoder, metadata = self.load_model(model_filename)
|
||||
result["valid"] = True
|
||||
|
||||
# Check metadata
|
||||
if not metadata or metadata.get("model_type") == "unknown":
|
||||
result["warnings"].append("Không có metadata hoặc metadata không đầy đủ")
|
||||
|
||||
# Check required features
|
||||
if not metadata.get("features"):
|
||||
result["warnings"].append("Danh sách features không có trong metadata")
|
||||
|
||||
# Check model object
|
||||
if model is None:
|
||||
result["errors"].append("Model object is None")
|
||||
result["valid"] = False
|
||||
|
||||
except Exception as e:
|
||||
result["errors"].append(f"Lỗi khi load model: {str(e)}")
|
||||
result["valid"] = False
|
||||
|
||||
return result
|
||||
|
||||
def get_required_features(self, model_filename: str) -> List[str]:
|
||||
"""
|
||||
Lấy danh sách features cần thiết cho model
|
||||
|
||||
Returns:
|
||||
List of feature names
|
||||
"""
|
||||
metadata = self._load_metadata(model_filename)
|
||||
return metadata.get("features", [])
|
||||
|
||||
def predict(self, model_filename: str, X: np.ndarray) -> np.ndarray:
|
||||
"""
|
||||
Predict using specified model
|
||||
|
||||
Args:
|
||||
model_filename: Model file name
|
||||
X: Features array (n_samples, n_features)
|
||||
|
||||
Returns:
|
||||
Predictions array
|
||||
"""
|
||||
if self.current_model is None or model_filename != getattr(self, '_current_model_filename', None):
|
||||
model, encoder, metadata = self.load_model(model_filename)
|
||||
self._current_model_filename = model_filename
|
||||
else:
|
||||
model = self.current_model
|
||||
metadata = self.current_metadata
|
||||
|
||||
# Validate input features
|
||||
expected_features = metadata.get("n_features", 0)
|
||||
if X.shape[1] != expected_features:
|
||||
raise ValueError(f"Expected {expected_features} features, got {X.shape[1]}")
|
||||
|
||||
# Predict
|
||||
predictions = model.predict(X)
|
||||
|
||||
return predictions
|
||||
|
||||
def get_model_info(self, model_filename: str) -> Dict[str, Any]:
|
||||
"""Get detailed info about a model"""
|
||||
models = self.list_models()
|
||||
for model in models:
|
||||
if model["filename"] == model_filename:
|
||||
return model
|
||||
return None
|
||||
|
||||
def delete_model(self, model_filename: str) -> bool:
|
||||
"""
|
||||
Xóa model và metadata
|
||||
|
||||
Returns:
|
||||
True if successful
|
||||
"""
|
||||
model_path = self.models_dir / model_filename
|
||||
|
||||
if not model_path.exists():
|
||||
return False
|
||||
|
||||
# Delete model file
|
||||
model_path.unlink()
|
||||
|
||||
# Delete metadata file if exists
|
||||
metadata_file = model_path.with_suffix('.json')
|
||||
if metadata_file.exists():
|
||||
metadata_file.unlink()
|
||||
|
||||
# Try alternative metadata file name
|
||||
metadata_file_alt = model_path.parent / (model_path.stem + "_info.json")
|
||||
if metadata_file_alt.exists():
|
||||
metadata_file_alt.unlink()
|
||||
|
||||
return True
|
||||
|
||||
def get_latest_model(self, model_type: Optional[str] = None) -> Optional[str]:
|
||||
"""
|
||||
Lấy model mới nhất (theo thời gian modified)
|
||||
|
||||
Args:
|
||||
model_type: Filter by model type (xgboost, cnn, etc.), None for any
|
||||
|
||||
Returns:
|
||||
Model filename or None
|
||||
"""
|
||||
models = self.list_models()
|
||||
|
||||
if model_type:
|
||||
models = [m for m in models if m.get("model_type") == model_type]
|
||||
|
||||
if not models:
|
||||
return None
|
||||
|
||||
# Already sorted by modified time
|
||||
return models[0]["filename"]
|
||||
|
||||
|
||||
# Singleton instance
|
||||
_model_manager = None
|
||||
|
||||
def get_model_manager() -> ModelManager:
|
||||
"""Get singleton ModelManager instance"""
|
||||
global _model_manager
|
||||
if _model_manager is None:
|
||||
_model_manager = ModelManager()
|
||||
return _model_manager
|
||||
Reference in New Issue
Block a user