clossers/didi-lot1-ai
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

LOT 1 - Optimizare script build -Instalare mono comanda

Browse source
This commit is contained in:
Dezvoltari Evotech 2026-06-27 06:42:02 -07:00
parent 5380c3fc63
commit 42ff22bf85
127 changed files with 16163 additions and 532 deletions
Download patch file Download diff file Expand all files Collapse all files

0
ai_platform/modules/domain_check/api/app/routes/__init__.py Normal file
View file

14
ai_platform/modules/domain_check/api/app/routes/batch.py Normal file
View file

@ -0,0 +1,14 @@
"""Batch routes - stub for now"""
from flask import Blueprint, jsonify
batch_bp = Blueprint('batch', __name__)
@batch_bp.route('/check/batch', methods=['POST'])
def batch_check():
"""Batch domain check - TODO: implement"""
return jsonify({'message': 'Not yet implemented'}), 501
@batch_bp.route('/batch/<string:batch_id>/status', methods=['GET'])
def batch_status(batch_id):
"""Get batch status - TODO: implement"""
return jsonify({'message': 'Not yet implemented', 'batch_id': batch_id}), 501

560
ai_platform/modules/domain_check/api/app/routes/check.py Normal file
View file

@ -0,0 +1,560 @@
"""
Domain Check Routes - Comprehensive Domain Verification API
"""
import uuid
import logging
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from flask import request, current_app
from flask_restx import Namespace, Resource, fields
from werkzeug.exceptions import BadRequest, HTTPException
from app import db
from app.models import Domain, WhoisRecord, DnsRecord, SslCertificate, RiskAssessment, CheckHistory
from app.services.whois_service import WhoisService
from app.services.risk_scorer import _coerce_datetime
from app.services.dns_service import DNSService
from app.services.ssl_service import SSLService
from app.services.risk_scorer import RiskScorer
from app.services.ip_intelligence_service import IPIntelligenceService
from app.services.http_analysis_service import HTTPAnalysisService
from app.services.blacklist_service import BlacklistService
from app.services.subdomain_service import SubdomainService
from app.services.port_scan_service import PortScanService
from app.services.mail_intelligence_service import MailIntelligenceService
logger = logging.getLogger(__name__)
# Create namespace
api = Namespace('check', description='Domain checking operations')
# Import model definitions from api_models.py
from app.api_models import create_api_models
# Create placeholder models
_temp_api = Namespace('_temp')
_models = create_api_models(_temp_api)
# Get the models we need for decorators
check_request_model = _models.get('domain_check_request')
check_response_model = _models.get('domain_check_response')
error_response_model = _models.get('error_response')
@api.route('/check')
class DomainCheck(Resource):
"""Domain Check Resource - Comprehensive domain verification"""
@api.doc(
'check_domain',
description='''Perform comprehensive domain verification including:
**Basic Checks:**
- WHOIS lookup (domain age, registrar, status, name servers)
- DNS record checking (A, AAAA, MX, TXT, NS, CNAME, SOA)
- SSL certificate validation (validity, issuer, expiration)
**Advanced Checks (when enabled):**
- IP Intelligence (geolocation, ASN, reverse DNS, hosting info)
- HTTP Security Analysis (headers, technology detection)
- Blacklist/Reputation checking (DNSBL, spam lists)
- Port Scanning (open ports, dangerous services)
- Subdomain Enumeration (Certificate Transparency)
Returns detailed domain information and comprehensive risk score (0-100).'''
)
@api.expect(check_request_model, validate=False)
@api.response(200, 'Success - Domain check completed', check_response_model)
@api.response(400, 'Bad Request - Invalid domain or parameters', error_response_model)
@api.response(500, 'Internal Server Error - Check failed', error_response_model)
def post(self):
"""
Check a single domain - Full comprehensive analysis
"""
start_time = time.time()
try:
# Get request data
try:
data = api.payload or {}
except BadRequest:
data = {}
if not data or 'domain' not in data:
return {
'success': False,
'error': {
'code': 'INVALID_REQUEST',
'message': 'Domain parameter is required',
'status': 400
}
}, 400
domain_name = data['domain'].lower().strip()
check_options = data.get('check_options', {
'whois': True,
'dns': True,
'ssl': True,
'ip_intelligence': True,
'http_analysis': True,
'blacklist': True,
'port_scan': False, # Disabled by default (slow)
'subdomains': False, # Disabled by default (slow)
'force_refresh': False
})
logger.info(f'Checking domain: {domain_name} with options: {check_options}')
# Generate check ID
check_id = uuid.uuid4()
# Get or create domain
domain, created = Domain.get_or_create(domain_name)
# Initialize all services
whoxy_api_key = current_app.config.get('WHOXY_API_KEY')
whois_service = WhoisService(whoxy_api_key=whoxy_api_key)
dns_service = DNSService()
ssl_service = SSLService()
ip_service = IPIntelligenceService()
http_service = HTTPAnalysisService()
blacklist_service = BlacklistService()
subdomain_service = SubdomainService()
port_scan_service = PortScanService()
mail_service = MailIntelligenceService()
risk_scorer = RiskScorer()
# Collect all domain data for risk scoring
domain_data = {
'domain': domain_name,
'whois': None,
'dns': None,
'ssl': None,
'ip_intelligence': None,
'http_analysis': None,
'blacklist': None,
'port_scan': None,
'subdomains': None
}
# Response data
response_data = {
'domain': domain_name,
'check_id': str(check_id),
'timestamp': datetime.utcnow().isoformat() + 'Z'
}
# Primary IP for subsequent checks
primary_ip = None
# ----------------------------------------------------------------
# Concurrent fetch. Network lookups are I/O-bound and independent,
# so we run them in a thread pool instead of sequentially (cuts a
# full check from ~sum-of-calls to ~max-of-calls). DB writes stay in
# the main thread afterwards because the SQLAlchemy session is not
# thread-safe. Each task runs inside an app context so services that
# read current_app keep working off the main thread.
# ----------------------------------------------------------------
app_obj = current_app._get_current_object()
def _run(fn, *a, **kw):
with app_obj.app_context():
try:
return fn(*a, **kw)
except Exception as exc:
logger.warning(f'{getattr(fn, "__name__", fn)} failed: {exc}')
return None
# Phase 1 — tasks that only need the domain name.
p1 = {}
with ThreadPoolExecutor(max_workers=5) as ex:
if check_options.get('whois', True):
p1['whois'] = ex.submit(_run, whois_service.lookup, domain_name)
if check_options.get('dns', True):
p1['dns'] = ex.submit(_run, dns_service.lookup, domain_name)
if check_options.get('ssl', True):
p1['ssl'] = ex.submit(_run, ssl_service.check, domain_name)
if check_options.get('http_analysis', True):
p1['http'] = ex.submit(_run, http_service.analyze, domain_name)
if check_options.get('subdomains', False):
extra_subs = check_options.get('extra_subdomains') or []
p1['subdomains'] = ex.submit(_run, subdomain_service.enumerate,
get_root_domain(domain_name), extra_subs)
whois_data = p1['whois'].result() if 'whois' in p1 else None
dns_data = p1['dns'].result() if 'dns' in p1 else None
ssl_data = p1['ssl'].result() if 'ssl' in p1 else None
http_data = p1['http'].result() if 'http' in p1 else None
subdomain_data = p1['subdomains'].result() if 'subdomains' in p1 else None
if dns_data and dns_data.get('a_records'):
primary_ip = dns_data['a_records'][0]
# Phase 2 — tasks that need DNS results (primary IP, MX/TXT).
p2 = {}
with ThreadPoolExecutor(max_workers=4) as ex:
if check_options.get('ip_intelligence', True) and primary_ip:
p2['ip'] = ex.submit(_run, ip_service.lookup, primary_ip)
if check_options.get('blacklist', True):
p2['blacklist'] = ex.submit(_run, blacklist_service.check_all, domain_name, primary_ip)
if check_options.get('port_scan', False) and primary_ip:
p2['port'] = ex.submit(_run, port_scan_service.quick_scan, primary_ip)
if check_options.get('mail', True):
mx = dns_data.get('mx_records') if dns_data else None
txt = dns_data.get('txt_records') if dns_data else None
p2['mail'] = ex.submit(_run, mail_service.analyze, domain_name, mx, txt,
check_options.get('smtp_probe', False))
ip_data = p2['ip'].result() if 'ip' in p2 else None
blacklist_data = p2['blacklist'].result() if 'blacklist' in p2 else None
port_data = p2['port'].result() if 'port' in p2 else None
mail_data = p2['mail'].result() if 'mail' in p2 else None
# ----------------------------------------------------------------
# Process results + persist (single-threaded, ordered).
# ----------------------------------------------------------------
# 1. WHOIS
if whois_data:
domain_data['whois'] = whois_data
response_data['whois'] = format_whois_response(whois_data)
# Save WHOIS record
try:
whois_record = WhoisRecord(
domain_id=domain.id,
creation_date=whois_data.get('creation_date'),
expiration_date=whois_data.get('expiration_date'),
updated_date=whois_data.get('updated_date'),
registrar=whois_data.get('registrar'),
registrar_url=whois_data.get('registrar_url'),
registrant_org=whois_data.get('registrant_org'),
registrant_country=whois_data.get('registrant_country'),
admin_email=whois_data.get('admin_email'),
name_servers=whois_data.get('name_servers', []),
status=whois_data.get('status', []),
dnssec=whois_data.get('dnssec'),
raw_whois_data=whois_data.get('raw_data', {}),
data_source=whois_data.get('data_source', 'rdap')
)
db.session.add(whois_record)
except Exception as e:
logger.warning(f'Failed to save WHOIS record: {e}')
# 2. DNS
if dns_data:
domain_data['dns'] = dns_data
response_data['dns'] = format_dns_response(dns_data)
# Save DNS records
try:
for record_type in ['a_records', 'aaaa_records', 'mx_records', 'txt_records', 'ns_records', 'cname_records']:
records = dns_data.get(record_type, [])
if records:
for record in records:
if isinstance(record, dict): # MX records
dns_record = DnsRecord(
domain_id=domain.id,
record_type='MX',
record_value=record['host'],
priority=record['priority']
)
else:
dns_record = DnsRecord(
domain_id=domain.id,
record_type=record_type.replace('_records', '').upper(),
record_value=str(record)
)
db.session.add(dns_record)
except Exception as e:
logger.warning(f'Failed to save DNS records: {e}')
# Registration / availability verdict (WHOIS + DNS combined signals)
if check_options.get('whois', True) or check_options.get('dns', True):
availability = determine_availability(whois_data, dns_data)
response_data['availability'] = availability
if 'whois' in response_data:
response_data['whois']['is_registered'] = availability['is_registered']
response_data['whois']['is_available'] = availability['is_available']
# 3. SSL Certificate
if ssl_data:
domain_data['ssl'] = ssl_data
response_data['ssl'] = format_ssl_response(ssl_data)
if ssl_data.get('has_ssl'):
try:
ssl_cert = SslCertificate(
domain_id=domain.id,
issuer=ssl_data.get('issuer'),
subject=ssl_data.get('subject'),
valid_from=ssl_data.get('valid_from'),
valid_until=ssl_data.get('valid_until'),
serial_number=ssl_data.get('serial_number'),
signature_algorithm=ssl_data.get('signature_algorithm'),
key_size=ssl_data.get('key_size'),
is_wildcard=ssl_data.get('is_wildcard', False),
is_self_signed=ssl_data.get('is_self_signed', False),
is_valid=ssl_data.get('is_valid', True)
)
db.session.add(ssl_cert)
except Exception as e:
logger.warning(f'Failed to save SSL certificate: {e}')
# 4. IP Intelligence
if ip_data:
ip_data['hosting_score'] = ip_service.get_hosting_score(ip_data)
domain_data['ip_intelligence'] = ip_data
response_data['ip_intelligence'] = ip_data
# 5. HTTP Analysis
if http_data:
domain_data['http_analysis'] = http_data
response_data['http_analysis'] = http_data
# 6. Blacklist
if blacklist_data:
domain_data['blacklist'] = blacklist_data
response_data['blacklist'] = blacklist_data
# 7. Mail Intelligence (email infrastructure deep analysis)
if mail_data:
domain_data['mail'] = mail_data
response_data['mail'] = mail_data
# 8. Port Scan (optional)
if port_data:
domain_data['port_scan'] = port_data
response_data['port_scan'] = port_data
# 9. Subdomain Enumeration (optional)
if subdomain_data:
domain_data['subdomains'] = subdomain_data
response_data['subdomains'] = subdomain_data
# 9. Calculate Risk Score
logger.info(f'Calculating risk score for {domain_name}')
risk_assessment_result = risk_scorer.calculate_risk(domain_data)
# Save risk assessment
try:
risk_assessment = RiskAssessment(
domain_id=domain.id,
check_id=check_id,
total_score=risk_assessment_result['total_score'],
risk_level=risk_assessment_result['risk_level'],
domain_age_score=risk_assessment_result['individual_scores'].get('domain_age', 0),
domain_age_days=next((f['details'].get('age_days') for f in risk_assessment_result['factors'] if f['factor'] == 'domain_age'), None),
ssl_score=risk_assessment_result['individual_scores'].get('ssl', 0),
dns_score=risk_assessment_result['individual_scores'].get('dns', 0),
reputation_score=risk_assessment_result['individual_scores'].get('blacklist', 0),
whois_score=risk_assessment_result['individual_scores'].get('whois', 0),
factors=risk_assessment_result['factors'],
is_new_domain=risk_assessment_result['is_new_domain'],
is_suspicious=risk_assessment_result['is_suspicious'],
requires_manual_review=risk_assessment_result['requires_manual_review']
)
db.session.add(risk_assessment)
except Exception as e:
logger.warning(f'Failed to save risk assessment: {e}')
# Save check history
processing_time_ms = int((time.time() - start_time) * 1000)
try:
check_history = CheckHistory(
check_id=check_id,
domain_id=domain.id,
requested_by='api',
request_ip=request.remote_addr,
user_agent=request.headers.get('User-Agent'),
check_options=check_options,
processing_time_ms=processing_time_ms,
cache_hit=False,
status='completed'
)
db.session.add(check_history)
except Exception as e:
logger.warning(f'Failed to save check history: {e}')
# Commit all changes
try:
db.session.commit()
except Exception as e:
logger.error(f'Database commit failed: {e}')
db.session.rollback()
# Build response
response_data['risk_score'] = {
'total': risk_assessment_result['total_score'],
'level': risk_assessment_result['risk_level'],
'factors': risk_assessment_result['factors'],
'formula_breakdown': risk_assessment_result.get('formula_breakdown', []),
'formula_string': risk_assessment_result.get('formula_string', ''),
'thresholds': risk_assessment_result.get('risk_thresholds', {
'low': '0-25',
'medium': '26-50',
'high': '51-75',
'critical': '76-100'
}),
'is_new_domain': risk_assessment_result['is_new_domain'],
'is_suspicious': risk_assessment_result['is_suspicious'],
'is_blacklisted': risk_assessment_result.get('is_blacklisted', False),
'requires_manual_review': risk_assessment_result['requires_manual_review']
}
response = {
'success': True,
'data': response_data,
'metadata': {
'cached': False,
'processing_time_ms': processing_time_ms,
'api_version': current_app.config.get('API_VERSION', 'v1'),
'checks_performed': [k for k, v in check_options.items() if v and k != 'force_refresh']
}
}
return response, 200
except HTTPException:
# Re-raise HTTP exceptions (400, 404, etc.) as-is
raise
except Exception as e:
logger.error(f'Error checking domain: {str(e)}', exc_info=True)
db.session.rollback()
api.abort(500, f'An error occurred while checking the domain: {str(e)}',
success=False,
error={
'code': 'INTERNAL_ERROR',
'message': f'An error occurred while checking the domain: {str(e)}'
})
def get_root_domain(domain: str) -> str:
"""Extract root domain from subdomain"""
parts = domain.split('.')
if len(parts) >= 2:
# Handle common TLDs
common_tlds = ['com', 'org', 'net', 'io', 'co', 'eu', 'ro', 'de', 'uk', 'fr']
if parts[-1] in common_tlds:
return '.'.join(parts[-2:])
# Handle country code TLDs like .co.uk
if len(parts) >= 3 and parts[-2] in ['co', 'com', 'org', 'net', 'gov']:
return '.'.join(parts[-3:])
return domain
def _iso(value):
"""Serialize a datetime to ISO-8601 Z, or None. Coerces stray strings safely."""
dt = _coerce_datetime(value)
return dt.isoformat() + 'Z' if dt else None
def format_whois_response(whois_data: dict) -> dict:
"""Format WHOIS data for API response"""
creation_date = _coerce_datetime(whois_data.get('creation_date'))
expiration_date = _coerce_datetime(whois_data.get('expiration_date'))
age_days = (datetime.utcnow() - creation_date).days if creation_date else None
days_until_expiry = (expiration_date - datetime.utcnow()).days if expiration_date else None
return {
'creation_date': _iso(whois_data.get('creation_date')),
'expiration_date': _iso(whois_data.get('expiration_date')),
'updated_date': _iso(whois_data.get('updated_date')),
'registrar': whois_data.get('registrar'),
'age_days': age_days,
'days_until_expiry': days_until_expiry,
'status': whois_data.get('status', []),
'name_servers': whois_data.get('name_servers', []),
'dnssec': whois_data.get('dnssec'),
'registrant_org': whois_data.get('registrant_org'),
'registrant_country': whois_data.get('registrant_country'),
'is_registered': whois_data.get('is_registered'),
'data_source': whois_data.get('data_source', 'whois')
}
def determine_availability(whois_data: dict, dns_data: dict) -> dict:
"""Combine WHOIS and DNS signals into an explicit registration verdict.
WHOIS alone is unreliable for sparse registries (ROTLD .ro), so DNS
resolution (NS/A records) is used as a strong corroborating signal.
"""
whois_signal = whois_data.get('is_registered') if whois_data else None
dns_has_records = False
if dns_data:
dns_has_records = bool(
dns_data.get('ns_records') or dns_data.get('a_records') or
dns_data.get('aaaa_records') or dns_data.get('mx_records')
)
signals = {
'whois_has_data': whois_signal is True,
'dns_resolves': dns_has_records,
}
# Decision: any positive signal => registered. Confidence is high when
# WHOIS and DNS agree, low when relying on a single weak signal.
if whois_signal is True or dns_has_records:
is_registered = True
confidence = 'high' if (whois_signal is True and dns_has_records) else 'medium'
elif whois_signal is False:
is_registered = False
confidence = 'high' if not dns_has_records else 'low'
else:
# WHOIS inconclusive (None) and no DNS records => probably available.
is_registered = False
confidence = 'low'
return {
'is_registered': is_registered,
'is_available': not is_registered,
'confidence': confidence,
'signals': signals
}
def format_dns_response(dns_data: dict) -> dict:
"""Format DNS data for API response"""
return {
'a_records': dns_data.get('a_records', []),
'aaaa_records': dns_data.get('aaaa_records', []),
'mx_records': dns_data.get('mx_records', []),
'txt_records': dns_data.get('txt_records', []),
'ns_records': dns_data.get('ns_records', []),
'cname_records': dns_data.get('cname_records', []),
'soa_record': dns_data.get('soa_record'),
'has_spf': dns_data.get('has_spf', False),
'has_dkim': dns_data.get('has_dkim', False),
'has_dmarc': dns_data.get('has_dmarc', False),
'spf_record': dns_data.get('spf_record'),
'dmarc_record': dns_data.get('dmarc_record')
}
def format_ssl_response(ssl_data: dict) -> dict:
"""Format SSL data for API response"""
if not ssl_data.get('has_ssl'):
return {
'has_ssl': False,
'error': ssl_data.get('error')
}
valid_from = ssl_data.get('valid_from')
valid_until = ssl_data.get('valid_until')
return {
'has_ssl': True,
'is_valid': ssl_data.get('is_valid', False),
'is_self_signed': ssl_data.get('is_self_signed', False),
'is_expired': ssl_data.get('is_expired', False),
'is_wildcard': ssl_data.get('is_wildcard', False),
'issuer': ssl_data.get('issuer'),
'subject': ssl_data.get('subject'),
'valid_from': valid_from.isoformat() + 'Z' if valid_from and hasattr(valid_from, 'isoformat') else str(valid_from) if valid_from else None,
'valid_until': valid_until.isoformat() + 'Z' if valid_until and hasattr(valid_until, 'isoformat') else str(valid_until) if valid_until else None,
'days_until_expiry': ssl_data.get('days_until_expiry'),
'key_size': ssl_data.get('key_size'),
'signature_algorithm': ssl_data.get('signature_algorithm'),
'san': ssl_data.get('san', [])
}

9
ai_platform/modules/domain_check/api/app/routes/domain.py Normal file
View file

@ -0,0 +1,9 @@
"""Domain routes - stub for now"""
from flask import Blueprint, jsonify
domain_bp = Blueprint('domain', __name__)
@domain_bp.route('/domain/<string:domain>', methods=['GET'])
def get_domain(domain):
"""Get domain details - TODO: implement"""
return jsonify({'message': 'Not yet implemented', 'domain': domain}), 501

9
ai_platform/modules/domain_check/api/app/routes/search.py Normal file
View file

@ -0,0 +1,9 @@
"""Search routes - stub for now"""
from flask import Blueprint, jsonify
search_bp = Blueprint('search', __name__)
@search_bp.route('/search', methods=['GET'])
def search_domains():
"""Search domains - TODO: implement"""
return jsonify({'message': 'Not yet implemented'}), 501

9
ai_platform/modules/domain_check/api/app/routes/stats.py Normal file
View file

@ -0,0 +1,9 @@
"""Stats routes - stub for now"""
from flask import Blueprint, jsonify
stats_bp = Blueprint('stats', __name__)
@stats_bp.route('/stats', methods=['GET'])
def get_stats():
"""Get system statistics - TODO: implement"""
return jsonify({'message': 'Not yet implemented'}), 501