LOT 1 - Optimizare script build -Instalare mono comanda
This commit is contained in:
parent
5380c3fc63
commit
42ff22bf85
127 changed files with 16163 additions and 532 deletions
68
ai_platform/modules/domain_check/api/Dockerfile
Normal file
68
ai_platform/modules/domain_check/api/Dockerfile
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
# Multi-stage build for Python API
|
||||
|
||||
# Stage 1: Builder
|
||||
FROM python:3.10-slim as builder
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /build
|
||||
|
||||
# Install system dependencies
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
g++ \
|
||||
make \
|
||||
libpq-dev \
|
||||
libssl-dev \
|
||||
libffi-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy requirements
|
||||
COPY requirements.txt .
|
||||
|
||||
# Create virtual environment and install dependencies
|
||||
RUN python -m venv /opt/venv
|
||||
ENV PATH="/opt/venv/bin:$PATH"
|
||||
RUN pip install --no-cache-dir --upgrade pip setuptools wheel && \
|
||||
pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Stage 2: Runtime
|
||||
FROM python:3.10-slim
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Install runtime dependencies only
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libpq5 \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy virtual environment from builder
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
|
||||
# Set environment variables
|
||||
ENV PATH="/opt/venv/bin:$PATH" \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
FLASK_APP=run.py
|
||||
|
||||
# Create non-root user
|
||||
RUN useradd -m -u 1000 appuser && \
|
||||
mkdir -p /app/logs && \
|
||||
chown -R appuser:appuser /app
|
||||
|
||||
# Copy application code
|
||||
COPY --chown=appuser:appuser . /app/
|
||||
|
||||
# Switch to non-root user
|
||||
USER appuser
|
||||
|
||||
# Expose port
|
||||
EXPOSE 5000
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
|
||||
CMD curl -f http://localhost:5000/health || exit 1
|
||||
|
||||
# Default command
|
||||
CMD ["gunicorn", "--bind", "0.0.0.0:5000", "--workers", "4", "--threads", "2", "--timeout", "120", "--worker-class", "gevent", "--access-logfile", "-", "--error-logfile", "-", "run:app"]
|
||||
422
ai_platform/modules/domain_check/api/app/__init__.py
Normal file
422
ai_platform/modules/domain_check/api/app/__init__.py
Normal file
|
|
@ -0,0 +1,422 @@
|
|||
"""
|
||||
Domain Check API - Application Factory
|
||||
"""
|
||||
import ipaddress
|
||||
import logging
|
||||
import os
|
||||
from flask import Flask, jsonify, request
|
||||
from flask_sqlalchemy import SQLAlchemy
|
||||
from flask_migrate import Migrate
|
||||
from flask_cors import CORS
|
||||
from flask_restx import Api
|
||||
from redis import Redis
|
||||
from celery import Celery
|
||||
|
||||
try:
|
||||
from flask_limiter import Limiter
|
||||
from flask_limiter.util import get_remote_address
|
||||
LIMITER_AVAILABLE = True
|
||||
except ImportError:
|
||||
LIMITER_AVAILABLE = False
|
||||
|
||||
from app.config import get_config
|
||||
|
||||
# Initialize extensions
|
||||
db = SQLAlchemy()
|
||||
migrate = Migrate()
|
||||
redis_client = None
|
||||
celery_app = Celery(__name__)
|
||||
limiter = Limiter(key_func=get_remote_address) if LIMITER_AVAILABLE else None
|
||||
|
||||
|
||||
def _is_internal_request() -> bool:
|
||||
"""Rate-limit exemption: never throttle internal LAN / loopback callers
|
||||
(the dashboard and server-to-server consumers like ZEUS) or health checks."""
|
||||
if request.path == '/health':
|
||||
return True
|
||||
try:
|
||||
ip = ipaddress.ip_address(get_remote_address())
|
||||
return ip.is_private or ip.is_loopback or ip.is_link_local
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def create_app(config_name=None):
|
||||
"""
|
||||
Application factory pattern
|
||||
|
||||
Args:
|
||||
config_name: Configuration name (development, production, testing)
|
||||
|
||||
Returns:
|
||||
Flask application instance
|
||||
"""
|
||||
if config_name is None:
|
||||
config_name = os.getenv('FLASK_ENV', 'development')
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
# Load configuration
|
||||
config = get_config(config_name)
|
||||
app.config.from_object(config)
|
||||
config.init_app(app)
|
||||
|
||||
# Initialize extensions
|
||||
init_extensions(app)
|
||||
|
||||
# Setup logging
|
||||
setup_logging(app)
|
||||
|
||||
# Register blueprints and routes
|
||||
register_blueprints(app)
|
||||
|
||||
# Register error handlers
|
||||
register_error_handlers(app)
|
||||
|
||||
# Setup Swagger/ReDoc API documentation
|
||||
setup_api_docs(app)
|
||||
|
||||
app.logger.info(f'Domain Check API started in {config_name} mode')
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def init_extensions(app):
|
||||
"""Initialize Flask extensions"""
|
||||
global redis_client
|
||||
|
||||
# Database
|
||||
db.init_app(app)
|
||||
migrate.init_app(app, db)
|
||||
|
||||
# CORS
|
||||
CORS(app,
|
||||
origins=app.config['CORS_ORIGINS'],
|
||||
allow_headers=app.config['CORS_ALLOW_HEADERS'],
|
||||
methods=app.config['CORS_METHODS'])
|
||||
|
||||
# Redis
|
||||
try:
|
||||
redis_client = Redis.from_url(
|
||||
app.config['REDIS_URL'],
|
||||
decode_responses=True,
|
||||
socket_timeout=5,
|
||||
socket_connect_timeout=5
|
||||
)
|
||||
redis_client.ping()
|
||||
app.logger.info('Redis connection established')
|
||||
except Exception as e:
|
||||
app.logger.warning(f'Redis connection failed: {e}. Caching disabled.')
|
||||
redis_client = None
|
||||
|
||||
# Celery
|
||||
init_celery(app)
|
||||
|
||||
# Rate limiting (protects paid WHOIS/VirusTotal quotas from external abuse).
|
||||
# Internal LAN/loopback traffic is exempt so it never throttles ZEUS or the
|
||||
# dashboard. Storage errors are swallowed so a Redis hiccup can't 500 the API.
|
||||
if limiter is not None:
|
||||
try:
|
||||
app.config.setdefault('RATELIMIT_STORAGE_URI', app.config['REDIS_URL'])
|
||||
app.config.setdefault('RATELIMIT_DEFAULT', os.getenv('RATELIMIT_DEFAULT', '240 per minute;5000 per hour'))
|
||||
app.config.setdefault('RATELIMIT_HEADERS_ENABLED', True)
|
||||
app.config.setdefault('RATELIMIT_SWALLOW_ERRORS', True)
|
||||
limiter.init_app(app)
|
||||
limiter.request_filter(_is_internal_request)
|
||||
app.logger.info('Rate limiter enabled (internal traffic exempt)')
|
||||
except Exception as e:
|
||||
app.logger.warning(f'Rate limiter init failed, continuing without it: {e}')
|
||||
|
||||
# Store redis_client in app context
|
||||
app.redis = redis_client
|
||||
|
||||
|
||||
def init_celery(app):
|
||||
"""Initialize Celery"""
|
||||
celery_app.conf.update(
|
||||
broker_url=app.config['CELERY_BROKER_URL'],
|
||||
result_backend=app.config['CELERY_RESULT_BACKEND'],
|
||||
task_serializer=app.config['CELERY_TASK_SERIALIZER'],
|
||||
result_serializer=app.config['CELERY_RESULT_SERIALIZER'],
|
||||
accept_content=app.config['CELERY_ACCEPT_CONTENT'],
|
||||
timezone=app.config['CELERY_TIMEZONE'],
|
||||
enable_utc=app.config['CELERY_ENABLE_UTC'],
|
||||
task_track_started=app.config['CELERY_TASK_TRACK_STARTED'],
|
||||
task_time_limit=app.config['CELERY_TASK_TIME_LIMIT'],
|
||||
task_soft_time_limit=app.config['CELERY_TASK_SOFT_TIME_LIMIT']
|
||||
)
|
||||
|
||||
class ContextTask(celery_app.Task):
|
||||
"""Make celery tasks work with Flask app context"""
|
||||
def __call__(self, *args, **kwargs):
|
||||
with app.app_context():
|
||||
return self.run(*args, **kwargs)
|
||||
|
||||
celery_app.Task = ContextTask
|
||||
app.celery = celery_app
|
||||
return celery_app
|
||||
|
||||
|
||||
def setup_logging(app):
|
||||
"""Configure application logging"""
|
||||
log_level = getattr(logging, app.config['LOG_LEVEL'].upper(), logging.INFO)
|
||||
|
||||
# Root logger
|
||||
logging.basicConfig(
|
||||
level=log_level,
|
||||
format=app.config['LOG_FORMAT']
|
||||
)
|
||||
|
||||
# App logger
|
||||
app.logger.setLevel(log_level)
|
||||
|
||||
# Disable werkzeug request logs in production
|
||||
if not app.config['DEBUG']:
|
||||
logging.getLogger('werkzeug').setLevel(logging.WARNING)
|
||||
|
||||
|
||||
def register_blueprints(app):
|
||||
"""Register Flask blueprints"""
|
||||
|
||||
# Health check endpoint
|
||||
@app.route('/health')
|
||||
def health_check():
|
||||
"""Health check endpoint for Docker and load balancers"""
|
||||
# Simple health mode - minimal response (recommended for production)
|
||||
if app.config.get('SIMPLE_HEALTH', False):
|
||||
try:
|
||||
from sqlalchemy import text
|
||||
db.session.execute(text('SELECT 1'))
|
||||
return jsonify({'status': 'ok'}), 200
|
||||
except Exception:
|
||||
return jsonify({'status': 'error'}), 503
|
||||
|
||||
# Detailed health mode (development)
|
||||
health_status = {
|
||||
'status': 'healthy',
|
||||
'version': app.config['API_VERSION'],
|
||||
'environment': os.getenv('FLASK_ENV', 'development')
|
||||
}
|
||||
|
||||
# Check database
|
||||
try:
|
||||
from sqlalchemy import text
|
||||
db.session.execute(text('SELECT 1'))
|
||||
health_status['database'] = 'connected'
|
||||
except Exception as e:
|
||||
health_status['database'] = f'error: {str(e)}'
|
||||
health_status['status'] = 'unhealthy'
|
||||
|
||||
# Check Redis
|
||||
if app.redis:
|
||||
try:
|
||||
app.redis.ping()
|
||||
health_status['redis'] = 'connected'
|
||||
except Exception as e:
|
||||
health_status['redis'] = f'error: {str(e)}'
|
||||
else:
|
||||
health_status['redis'] = 'disabled'
|
||||
|
||||
status_code = 200 if health_status['status'] == 'healthy' else 503
|
||||
return jsonify(health_status), status_code
|
||||
|
||||
# Root endpoint - serve HTML dashboard
|
||||
@app.route('/')
|
||||
def index():
|
||||
"""Serve the domain check dashboard"""
|
||||
from flask import send_from_directory
|
||||
return send_from_directory('static', 'index.html')
|
||||
|
||||
# API info endpoint
|
||||
@app.route('/api')
|
||||
def api_info():
|
||||
"""API information endpoint"""
|
||||
response = {
|
||||
'name': app.config['API_TITLE'],
|
||||
'version': app.config['API_VERSION'],
|
||||
'endpoints': {
|
||||
'health': '/health',
|
||||
'api': f'/api/{app.config["API_VERSION"]}',
|
||||
'dashboard': '/'
|
||||
}
|
||||
}
|
||||
# Only show docs links if swagger is enabled
|
||||
if not app.config.get('DISABLE_SWAGGER', False):
|
||||
response['description'] = app.config['API_DESCRIPTION']
|
||||
response['documentation'] = {
|
||||
'swagger': '/docs',
|
||||
'redoc': '/redoc'
|
||||
}
|
||||
return jsonify(response)
|
||||
|
||||
# Note: Namespaces will be registered in setup_api_docs function
|
||||
|
||||
|
||||
def register_error_handlers(app):
|
||||
"""Register error handlers"""
|
||||
|
||||
@app.errorhandler(404)
|
||||
def not_found(error):
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': {
|
||||
'code': 'NOT_FOUND',
|
||||
'message': 'The requested resource was not found',
|
||||
'status': 404
|
||||
}
|
||||
}), 404
|
||||
|
||||
@app.errorhandler(500)
|
||||
def internal_error(error):
|
||||
app.logger.error(f'Internal server error: {error}')
|
||||
db.session.rollback()
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': {
|
||||
'code': 'INTERNAL_SERVER_ERROR',
|
||||
'message': 'An internal server error occurred',
|
||||
'status': 500
|
||||
}
|
||||
}), 500
|
||||
|
||||
@app.errorhandler(400)
|
||||
def bad_request(error):
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': {
|
||||
'code': 'BAD_REQUEST',
|
||||
'message': str(error),
|
||||
'status': 400
|
||||
}
|
||||
}), 400
|
||||
|
||||
@app.errorhandler(429)
|
||||
def rate_limit_exceeded(error):
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': {
|
||||
'code': 'RATE_LIMIT_EXCEEDED',
|
||||
'message': 'Too many requests. Please try again later.',
|
||||
'status': 429
|
||||
}
|
||||
}), 429
|
||||
|
||||
|
||||
def setup_api_docs(app):
|
||||
"""Setup Swagger/ReDoc API documentation"""
|
||||
|
||||
# Check if swagger should be disabled (production security)
|
||||
disable_swagger = app.config.get('DISABLE_SWAGGER', False)
|
||||
|
||||
# Create API instance - doc=False disables swagger UI
|
||||
api = Api(
|
||||
app,
|
||||
version=app.config['API_VERSION'],
|
||||
title=app.config['API_TITLE'],
|
||||
description=app.config['API_DESCRIPTION'] if not disable_swagger else '',
|
||||
doc='/docs' if not disable_swagger else False,
|
||||
prefix=f'/api/{app.config["API_VERSION"]}',
|
||||
contact=app.config['API_CONTACT'].get('name') if not disable_swagger else None,
|
||||
contact_email=app.config['API_CONTACT'].get('email') if not disable_swagger else None,
|
||||
license=app.config['API_LICENSE'].get('name') if not disable_swagger else None,
|
||||
license_url=app.config['API_LICENSE'].get('url') if not disable_swagger else None,
|
||||
terms_url=app.config.get('API_TERMS_OF_SERVICE') if not disable_swagger else None,
|
||||
ordered=True,
|
||||
validate=True
|
||||
)
|
||||
|
||||
if disable_swagger:
|
||||
app.logger.info('Swagger/ReDoc documentation disabled (DISABLE_SWAGGER=true)')
|
||||
|
||||
# Import and create API models
|
||||
from app.api_models import create_api_models
|
||||
models = create_api_models(api)
|
||||
|
||||
# Import and register namespaces
|
||||
try:
|
||||
from app.routes.check import api as check_ns
|
||||
|
||||
# Attach models to namespace
|
||||
check_ns.models.update(models)
|
||||
|
||||
# Register namespace
|
||||
api.add_namespace(check_ns, path='')
|
||||
|
||||
app.logger.info('API namespace registered successfully: check')
|
||||
except ImportError as e:
|
||||
app.logger.error(f'Failed to import check namespace: {e}')
|
||||
except Exception as e:
|
||||
app.logger.error(f'Failed to register check routes: {e}')
|
||||
|
||||
# Try to import other namespaces (if implemented)
|
||||
try:
|
||||
from app.routes.domain import api as domain_ns
|
||||
domain_ns.models.update(models)
|
||||
api.add_namespace(domain_ns, path='')
|
||||
app.logger.info('API namespace registered successfully: domain')
|
||||
except ImportError:
|
||||
app.logger.debug('domain namespace not yet implemented')
|
||||
|
||||
try:
|
||||
from app.routes.stats import api as stats_ns
|
||||
stats_ns.models.update(models)
|
||||
api.add_namespace(stats_ns, path='')
|
||||
app.logger.info('API namespace registered successfully: stats')
|
||||
except ImportError:
|
||||
app.logger.debug('stats namespace not yet implemented')
|
||||
|
||||
try:
|
||||
from app.routes.search import api as search_ns
|
||||
search_ns.models.update(models)
|
||||
api.add_namespace(search_ns, path='')
|
||||
app.logger.info('API namespace registered successfully: search')
|
||||
except ImportError:
|
||||
app.logger.debug('search namespace not yet implemented')
|
||||
|
||||
try:
|
||||
from app.routes.batch import api as batch_ns
|
||||
batch_ns.models.update(models)
|
||||
api.add_namespace(batch_ns, path='')
|
||||
app.logger.info('API namespace registered successfully: batch')
|
||||
except ImportError:
|
||||
app.logger.debug('batch namespace not yet implemented')
|
||||
|
||||
# Add ReDoc endpoint (only if swagger is enabled)
|
||||
if not disable_swagger:
|
||||
@app.route('/redoc')
|
||||
def redoc():
|
||||
"""ReDoc API documentation"""
|
||||
return f'''
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>{app.config['API_TITLE']} - ReDoc</title>
|
||||
<meta charset="utf-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<link href="https://fonts.googleapis.com/css?family=Montserrat:300,400,700|Roboto:300,400,700" rel="stylesheet">
|
||||
<style>
|
||||
body {{
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<redoc spec-url='/api/{app.config["API_VERSION"]}/swagger.json'></redoc>
|
||||
<script src="https://cdn.redoc.ly/redoc/latest/bundles/redoc.standalone.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
|
||||
app.api = api
|
||||
return api
|
||||
|
||||
|
||||
# Create celery app for worker
|
||||
def create_celery_app(app=None):
|
||||
"""
|
||||
Create and configure Celery app
|
||||
For use in worker process
|
||||
"""
|
||||
app = app or create_app()
|
||||
return app.celery
|
||||
424
ai_platform/modules/domain_check/api/app/api_models.py
Normal file
424
ai_platform/modules/domain_check/api/app/api_models.py
Normal file
|
|
@ -0,0 +1,424 @@
|
|||
"""
|
||||
Flask-RESTX API Models for Swagger Documentation
|
||||
"""
|
||||
from flask_restx import fields
|
||||
|
||||
|
||||
def create_api_models(api):
|
||||
"""
|
||||
Create and register all API models for Swagger documentation
|
||||
|
||||
Args:
|
||||
api: Flask-RESTX Api instance
|
||||
|
||||
Returns:
|
||||
Dictionary of model names to model objects
|
||||
"""
|
||||
|
||||
# ==================== REQUEST MODELS ====================
|
||||
|
||||
check_options_model = api.model('CheckOptions', {
|
||||
'whois': fields.Boolean(
|
||||
default=True,
|
||||
description='Perform WHOIS lookup',
|
||||
example=True
|
||||
),
|
||||
'dns': fields.Boolean(
|
||||
default=False,
|
||||
description='Perform DNS record lookup',
|
||||
example=True
|
||||
),
|
||||
'ssl': fields.Boolean(
|
||||
default=False,
|
||||
description='Check SSL certificate',
|
||||
example=False
|
||||
),
|
||||
'reputation': fields.Boolean(
|
||||
default=False,
|
||||
description='Check domain reputation (VirusTotal, blacklists)',
|
||||
example=False
|
||||
),
|
||||
'force_refresh': fields.Boolean(
|
||||
default=False,
|
||||
description='Force refresh, bypass cache',
|
||||
example=False
|
||||
)
|
||||
})
|
||||
|
||||
domain_check_request = api.model('DomainCheckRequest', {
|
||||
'domain': fields.String(
|
||||
required=True,
|
||||
description='Domain name to check',
|
||||
example='google.com',
|
||||
pattern=r'^[a-zA-Z0-9][a-zA-Z0-9-_.]*\.[a-zA-Z]{2,}$'
|
||||
),
|
||||
'check_options': fields.Nested(
|
||||
check_options_model,
|
||||
description='Options for what to check',
|
||||
required=False
|
||||
)
|
||||
})
|
||||
|
||||
# ==================== RESPONSE MODELS ====================
|
||||
|
||||
# WHOIS Response
|
||||
whois_response = api.model('WhoisResponse', {
|
||||
'creation_date': fields.String(
|
||||
description='Domain creation date (ISO 8601)',
|
||||
example='2024-01-15T10:30:00Z'
|
||||
),
|
||||
'expiration_date': fields.String(
|
||||
description='Domain expiration date (ISO 8601)',
|
||||
example='2026-01-15T10:30:00Z'
|
||||
),
|
||||
'registrar': fields.String(
|
||||
description='Domain registrar name',
|
||||
example='MarkMonitor Inc.'
|
||||
),
|
||||
'age_days': fields.Integer(
|
||||
description='Domain age in days',
|
||||
example=380
|
||||
),
|
||||
'status': fields.List(
|
||||
fields.String,
|
||||
description='Domain status codes',
|
||||
example=['clientDeleteProhibited', 'clientTransferProhibited']
|
||||
),
|
||||
'name_servers': fields.List(
|
||||
fields.String,
|
||||
description='Name servers',
|
||||
example=['ns1.google.com', 'ns2.google.com']
|
||||
)
|
||||
})
|
||||
|
||||
# DNS Response
|
||||
mx_record_model = api.model('MXRecord', {
|
||||
'priority': fields.Integer(description='MX priority', example=10),
|
||||
'host': fields.String(description='Mail server hostname', example='smtp.google.com')
|
||||
})
|
||||
|
||||
soa_record_model = api.model('SOARecord', {
|
||||
'mname': fields.String(description='Primary master name server', example='ns1.google.com'),
|
||||
'rname': fields.String(description='Responsible party email', example='dns-admin.google.com'),
|
||||
'serial': fields.Integer(description='Serial number', example=2024011501),
|
||||
'refresh': fields.Integer(description='Refresh interval', example=3600),
|
||||
'retry': fields.Integer(description='Retry interval', example=600),
|
||||
'expire': fields.Integer(description='Expire time', example=86400),
|
||||
'minimum': fields.Integer(description='Minimum TTL', example=300)
|
||||
})
|
||||
|
||||
dns_response = api.model('DNSResponse', {
|
||||
'a_records': fields.List(
|
||||
fields.String,
|
||||
description='A records (IPv4 addresses)',
|
||||
example=['142.250.185.46']
|
||||
),
|
||||
'aaaa_records': fields.List(
|
||||
fields.String,
|
||||
description='AAAA records (IPv6 addresses)',
|
||||
example=['2a00:1450:4001:801::200e']
|
||||
),
|
||||
'mx_records': fields.List(
|
||||
fields.Nested(mx_record_model),
|
||||
description='MX records (mail servers)'
|
||||
),
|
||||
'txt_records': fields.List(
|
||||
fields.String,
|
||||
description='TXT records',
|
||||
example=['v=spf1 include:_spf.google.com ~all']
|
||||
),
|
||||
'ns_records': fields.List(
|
||||
fields.String,
|
||||
description='NS records (name servers)',
|
||||
example=['ns1.google.com', 'ns2.google.com']
|
||||
),
|
||||
'cname_records': fields.List(
|
||||
fields.String,
|
||||
description='CNAME records',
|
||||
example=[]
|
||||
),
|
||||
'soa_record': fields.Nested(
|
||||
soa_record_model,
|
||||
description='SOA record (Start of Authority)'
|
||||
),
|
||||
'has_spf': fields.Boolean(
|
||||
description='Has SPF record',
|
||||
example=True
|
||||
),
|
||||
'has_dkim': fields.Boolean(
|
||||
description='Has DKIM record',
|
||||
example=True
|
||||
),
|
||||
'has_dmarc': fields.Boolean(
|
||||
description='Has DMARC record',
|
||||
example=True
|
||||
)
|
||||
})
|
||||
|
||||
# SSL Response
|
||||
ssl_response = api.model('SSLResponse', {
|
||||
'has_ssl': fields.Boolean(
|
||||
description='Has valid SSL certificate',
|
||||
example=True
|
||||
),
|
||||
'is_valid': fields.Boolean(
|
||||
description='Certificate is valid',
|
||||
example=True
|
||||
),
|
||||
'is_self_signed': fields.Boolean(
|
||||
description='Certificate is self-signed',
|
||||
example=False
|
||||
),
|
||||
'is_expired': fields.Boolean(
|
||||
description='Certificate is expired',
|
||||
example=False
|
||||
),
|
||||
'is_wildcard': fields.Boolean(
|
||||
description='Wildcard certificate',
|
||||
example=False
|
||||
),
|
||||
'issuer': fields.String(
|
||||
description='Certificate issuer',
|
||||
example='CN=GTS CA 1C3, O=Google Trust Services LLC, C=US'
|
||||
),
|
||||
'subject': fields.String(
|
||||
description='Certificate subject',
|
||||
example='CN=*.google.com'
|
||||
),
|
||||
'valid_from': fields.String(
|
||||
description='Valid from date (ISO 8601)',
|
||||
example='2024-12-01T08:15:00Z'
|
||||
),
|
||||
'valid_until': fields.String(
|
||||
description='Valid until date (ISO 8601)',
|
||||
example='2025-02-23T08:14:59Z'
|
||||
),
|
||||
'days_until_expiry': fields.Integer(
|
||||
description='Days until certificate expires',
|
||||
example=45
|
||||
),
|
||||
'key_size': fields.Integer(
|
||||
description='Key size in bits',
|
||||
example=2048
|
||||
),
|
||||
'signature_algorithm': fields.String(
|
||||
description='Signature algorithm',
|
||||
example='sha256WithRSAEncryption'
|
||||
),
|
||||
'error': fields.String(
|
||||
description='Error message if SSL check failed',
|
||||
example=None
|
||||
)
|
||||
})
|
||||
|
||||
# Risk Score Response
|
||||
risk_factor_model = api.model('RiskFactor', {
|
||||
'factor': fields.String(
|
||||
description='Risk factor name',
|
||||
example='domain_age',
|
||||
enum=['domain_age', 'reputation', 'ssl', 'dns', 'whois']
|
||||
),
|
||||
'score': fields.Float(
|
||||
description='Individual score for this factor',
|
||||
example=85.0
|
||||
),
|
||||
'weight': fields.Float(
|
||||
description='Weight of this factor',
|
||||
example=0.30
|
||||
),
|
||||
'weighted_score': fields.Float(
|
||||
description='Score * weight',
|
||||
example=25.5
|
||||
),
|
||||
'reason': fields.String(
|
||||
description='Explanation of the score',
|
||||
example='Domain is only 45 days old (HIGH RISK)'
|
||||
),
|
||||
'age_days': fields.Integer(
|
||||
description='Domain age in days (only for domain_age factor)',
|
||||
example=45
|
||||
)
|
||||
})
|
||||
|
||||
risk_thresholds_model = api.model('RiskThresholds', {
|
||||
'low': fields.String(
|
||||
description='Low risk score range',
|
||||
example='0-30'
|
||||
),
|
||||
'medium': fields.String(
|
||||
description='Medium risk score range',
|
||||
example='31-60'
|
||||
),
|
||||
'high': fields.String(
|
||||
description='High risk score range',
|
||||
example='61-85'
|
||||
),
|
||||
'critical': fields.String(
|
||||
description='Critical risk score range',
|
||||
example='86-100'
|
||||
)
|
||||
})
|
||||
|
||||
risk_score_response = api.model('RiskScoreResponse', {
|
||||
'total': fields.Float(
|
||||
description='Total risk score (0-100)',
|
||||
example=65.5,
|
||||
min=0,
|
||||
max=100
|
||||
),
|
||||
'level': fields.String(
|
||||
description='Risk level',
|
||||
example='HIGH',
|
||||
enum=['LOW', 'MEDIUM', 'HIGH', 'CRITICAL']
|
||||
),
|
||||
'factors': fields.List(
|
||||
fields.Nested(risk_factor_model),
|
||||
description='Detailed breakdown of risk factors'
|
||||
),
|
||||
'thresholds': fields.Nested(
|
||||
risk_thresholds_model,
|
||||
description='Risk level thresholds'
|
||||
)
|
||||
})
|
||||
|
||||
# Metadata Response
|
||||
metadata_response = api.model('MetadataResponse', {
|
||||
'cached': fields.Boolean(
|
||||
description='Response served from cache',
|
||||
example=False
|
||||
),
|
||||
'processing_time_ms': fields.Integer(
|
||||
description='Processing time in milliseconds',
|
||||
example=1250
|
||||
),
|
||||
'api_version': fields.String(
|
||||
description='API version',
|
||||
example='v1'
|
||||
)
|
||||
})
|
||||
|
||||
# Main Check Response Data
|
||||
check_response_data = api.model('CheckResponseData', {
|
||||
'domain': fields.String(
|
||||
description='Checked domain name',
|
||||
example='google.com'
|
||||
),
|
||||
'check_id': fields.String(
|
||||
description='Unique check ID (UUID)',
|
||||
example='123e4567-e89b-12d3-a456-426614174000'
|
||||
),
|
||||
'timestamp': fields.String(
|
||||
description='Check timestamp (ISO 8601)',
|
||||
example='2026-01-30T12:34:56Z'
|
||||
),
|
||||
'whois': fields.Nested(
|
||||
whois_response,
|
||||
description='WHOIS lookup results',
|
||||
allow_null=True
|
||||
),
|
||||
'dns': fields.Nested(
|
||||
dns_response,
|
||||
description='DNS lookup results',
|
||||
allow_null=True
|
||||
),
|
||||
'ssl': fields.Nested(
|
||||
ssl_response,
|
||||
description='SSL certificate check results',
|
||||
allow_null=True
|
||||
),
|
||||
'reputation': fields.Raw(
|
||||
description='Reputation check results (not implemented)',
|
||||
example=None
|
||||
),
|
||||
'risk_score': fields.Nested(
|
||||
risk_score_response,
|
||||
description='Risk assessment results',
|
||||
required=True
|
||||
)
|
||||
})
|
||||
|
||||
# Main Check Response
|
||||
domain_check_response = api.model('DomainCheckResponse', {
|
||||
'success': fields.Boolean(
|
||||
description='Request success status',
|
||||
example=True
|
||||
),
|
||||
'data': fields.Nested(
|
||||
check_response_data,
|
||||
description='Response data',
|
||||
required=True
|
||||
),
|
||||
'metadata': fields.Nested(
|
||||
metadata_response,
|
||||
description='Request metadata',
|
||||
required=True
|
||||
)
|
||||
})
|
||||
|
||||
# ==================== ERROR MODELS ====================
|
||||
|
||||
error_detail_model = api.model('ErrorDetail', {
|
||||
'code': fields.String(
|
||||
description='Error code',
|
||||
example='INVALID_REQUEST',
|
||||
enum=['INVALID_REQUEST', 'NOT_FOUND', 'INTERNAL_ERROR', 'RATE_LIMIT_EXCEEDED']
|
||||
),
|
||||
'message': fields.String(
|
||||
description='Error message',
|
||||
example='Domain parameter is required'
|
||||
),
|
||||
'status': fields.Integer(
|
||||
description='HTTP status code',
|
||||
example=400
|
||||
)
|
||||
})
|
||||
|
||||
error_response = api.model('ErrorResponse', {
|
||||
'success': fields.Boolean(
|
||||
description='Request success status',
|
||||
example=False
|
||||
),
|
||||
'error': fields.Nested(
|
||||
error_detail_model,
|
||||
description='Error details',
|
||||
required=True
|
||||
)
|
||||
})
|
||||
|
||||
# ==================== HEALTH CHECK MODEL ====================
|
||||
|
||||
health_response = api.model('HealthResponse', {
|
||||
'status': fields.String(
|
||||
description='Health status',
|
||||
example='healthy',
|
||||
enum=['healthy', 'unhealthy']
|
||||
),
|
||||
'version': fields.String(
|
||||
description='API version',
|
||||
example='v1'
|
||||
),
|
||||
'environment': fields.String(
|
||||
description='Environment name',
|
||||
example='production'
|
||||
),
|
||||
'database': fields.String(
|
||||
description='Database connection status',
|
||||
example='connected'
|
||||
),
|
||||
'redis': fields.String(
|
||||
description='Redis connection status',
|
||||
example='connected'
|
||||
)
|
||||
})
|
||||
|
||||
return {
|
||||
'domain_check_request': domain_check_request,
|
||||
'domain_check_response': domain_check_response,
|
||||
'error_response': error_response,
|
||||
'health_response': health_response,
|
||||
'check_options_model': check_options_model,
|
||||
'whois_response': whois_response,
|
||||
'dns_response': dns_response,
|
||||
'ssl_response': ssl_response,
|
||||
'risk_score_response': risk_score_response
|
||||
}
|
||||
6
ai_platform/modules/domain_check/api/app/celery_app.py
Normal file
6
ai_platform/modules/domain_check/api/app/celery_app.py
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
"""
|
||||
Celery Application for Domain Check
|
||||
"""
|
||||
from app import create_celery_app
|
||||
|
||||
celery = create_celery_app()
|
||||
234
ai_platform/modules/domain_check/api/app/config.py
Normal file
234
ai_platform/modules/domain_check/api/app/config.py
Normal file
|
|
@ -0,0 +1,234 @@
|
|||
"""
|
||||
Configuration module for Domain Check API
|
||||
"""
|
||||
import os
|
||||
from datetime import timedelta
|
||||
from typing import Dict, Any
|
||||
|
||||
|
||||
class Config:
|
||||
"""Base configuration"""
|
||||
|
||||
# Flask
|
||||
SECRET_KEY = os.getenv('SECRET_KEY', 'dev-secret-key-change-in-production')
|
||||
DEBUG = False
|
||||
TESTING = False
|
||||
|
||||
# Database
|
||||
SQLALCHEMY_DATABASE_URI = os.getenv(
|
||||
'DATABASE_URL',
|
||||
'postgresql://dns_admin:password@localhost:5432/domain_check'
|
||||
)
|
||||
SQLALCHEMY_TRACK_MODIFICATIONS = False
|
||||
SQLALCHEMY_ECHO = False
|
||||
SQLALCHEMY_POOL_SIZE = 10
|
||||
SQLALCHEMY_POOL_TIMEOUT = 30
|
||||
SQLALCHEMY_POOL_RECYCLE = 3600
|
||||
SQLALCHEMY_MAX_OVERFLOW = 20
|
||||
|
||||
# Redis
|
||||
REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0')
|
||||
REDIS_TTL_HOT = int(os.getenv('REDIS_TTL_HOT', 21600)) # 6 hours
|
||||
REDIS_TTL_WARM = int(os.getenv('REDIS_TTL_WARM', 86400)) # 24 hours
|
||||
REDIS_TTL_COLD = int(os.getenv('REDIS_TTL_COLD', 604800)) # 7 days
|
||||
|
||||
# Celery
|
||||
CELERY_BROKER_URL = os.getenv('CELERY_BROKER_URL', 'redis://localhost:6379/1')
|
||||
CELERY_RESULT_BACKEND = os.getenv('CELERY_RESULT_BACKEND', 'redis://localhost:6379/2')
|
||||
CELERY_TASK_SERIALIZER = 'json'
|
||||
CELERY_RESULT_SERIALIZER = 'json'
|
||||
CELERY_ACCEPT_CONTENT = ['json']
|
||||
CELERY_TIMEZONE = 'UTC'
|
||||
CELERY_ENABLE_UTC = True
|
||||
CELERY_TASK_TRACK_STARTED = True
|
||||
CELERY_TASK_TIME_LIMIT = 300 # 5 minutes
|
||||
CELERY_TASK_SOFT_TIME_LIMIT = 240 # 4 minutes
|
||||
|
||||
# API Keys
|
||||
WHOXY_API_KEY = os.getenv('WHOXY_API_KEY', '')
|
||||
VIRUSTOTAL_API_KEY = os.getenv('VIRUSTOTAL_API_KEY', '')
|
||||
|
||||
# Risk Scoring Thresholds
|
||||
RISK_THRESHOLD_LOW = int(os.getenv('RISK_THRESHOLD_LOW', 30))
|
||||
RISK_THRESHOLD_MEDIUM = int(os.getenv('RISK_THRESHOLD_MEDIUM', 60))
|
||||
RISK_THRESHOLD_HIGH = int(os.getenv('RISK_THRESHOLD_HIGH', 85))
|
||||
|
||||
# Domain Age Thresholds (days)
|
||||
DOMAIN_AGE_CRITICAL = int(os.getenv('DOMAIN_AGE_CRITICAL', 90)) # 3 months
|
||||
DOMAIN_AGE_HIGH = int(os.getenv('DOMAIN_AGE_HIGH', 180)) # 6 months
|
||||
DOMAIN_AGE_MEDIUM = int(os.getenv('DOMAIN_AGE_MEDIUM', 365)) # 1 year
|
||||
|
||||
# API Rate Limiting
|
||||
WHOXY_DAILY_LIMIT = int(os.getenv('WHOXY_DAILY_LIMIT', 8000))
|
||||
VIRUSTOTAL_DAILY_LIMIT = int(os.getenv('VIRUSTOTAL_DAILY_LIMIT', 500))
|
||||
|
||||
# Batch Processing
|
||||
BATCH_CHUNK_SIZE = int(os.getenv('BATCH_CHUNK_SIZE', 50))
|
||||
BATCH_TIMEOUT_SECONDS = int(os.getenv('BATCH_TIMEOUT_SECONDS', 300))
|
||||
|
||||
# API Settings
|
||||
API_VERSION = os.getenv('API_VERSION', 'v1')
|
||||
API_TITLE = 'Domain Check API - Anti-Fake News Tool'
|
||||
API_DESCRIPTION = '''
|
||||
**Comprehensive Domain Verification & Risk Scoring API**
|
||||
|
||||
This API provides powerful tools for detecting potentially malicious or newly-registered domains
|
||||
commonly used in disinformation campaigns and fake news distribution.
|
||||
|
||||
## 🎯 Key Features
|
||||
|
||||
- **WHOIS Analysis**: Domain age, registrar information, privacy protection detection
|
||||
- **DNS Verification**: A, AAAA, MX, TXT, NS, CNAME, SOA records with email security (SPF, DKIM, DMARC)
|
||||
- **SSL Certificate Validation**: Certificate validity, issuer verification, expiration monitoring
|
||||
- **Risk Scoring**: Multi-factor algorithm (0-100) with detailed breakdown
|
||||
- **Comprehensive Database**: Full history tracking and audit trail
|
||||
|
||||
## 📊 Risk Levels
|
||||
|
||||
- **LOW (0-30)**: Established, trustworthy domains
|
||||
- **MEDIUM (31-60)**: Moderate risk, requires attention
|
||||
- **HIGH (61-85)**: Suspicious activity detected
|
||||
- **CRITICAL (86-100)**: Newly registered or highly suspicious domains
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
Use the `/check` endpoint to verify any domain:
|
||||
|
||||
```bash
|
||||
curl -X POST http://domain-check-api:11000/api/v1/check/check \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-d '{"domain": "example.com", "check_options": {"whois": true, "dns": true}}'
|
||||
```
|
||||
|
||||
## 📖 Documentation
|
||||
|
||||
- **Swagger UI**: Interactive API testing and documentation
|
||||
- **ReDoc**: Clean, responsive API reference
|
||||
- **Platformă**: modul T4 (evaluare credibilitate sursă) al platformei DIDI
|
||||
'''
|
||||
API_CONTACT = {
|
||||
'name': 'DIDI Domain Check',
|
||||
'email': 'support@didi.local'
|
||||
}
|
||||
API_LICENSE = {
|
||||
'name': 'MIT',
|
||||
'url': 'https://opensource.org/licenses/MIT'
|
||||
}
|
||||
API_TERMS_OF_SERVICE = ''
|
||||
|
||||
# CORS - use CORS_ORIGINS env var for production (comma-separated)
|
||||
# Example: CORS_ORIGINS=https://didi.example.ro,https://admin.didi.example.ro
|
||||
# Default '*' allows all origins (development only!)
|
||||
_cors_origins_env = os.getenv('CORS_ORIGINS', '*')
|
||||
CORS_ORIGINS = ['*'] if _cors_origins_env == '*' else [o.strip() for o in _cors_origins_env.split(',') if o.strip()]
|
||||
CORS_ALLOW_HEADERS = ['Content-Type', 'Authorization']
|
||||
CORS_METHODS = ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS']
|
||||
|
||||
# Security - Documentation endpoints
|
||||
# Set DISABLE_SWAGGER=true in production to hide /docs and /redoc
|
||||
DISABLE_SWAGGER = os.getenv('DISABLE_SWAGGER', 'false').lower() == 'true'
|
||||
|
||||
# Health endpoint - set SIMPLE_HEALTH=true to return only {"status":"ok"}
|
||||
SIMPLE_HEALTH = os.getenv('SIMPLE_HEALTH', 'false').lower() == 'true'
|
||||
|
||||
# Logging
|
||||
LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO')
|
||||
LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
LOG_FILE = os.path.join(os.path.dirname(__file__), '..', 'logs', 'api.log')
|
||||
|
||||
# External Service URLs
|
||||
WHOXY_API_URL = 'https://api.whoxy.com/'
|
||||
VIRUSTOTAL_API_URL = 'https://www.virustotal.com/api/v3/'
|
||||
RDAP_BOOTSTRAP_URL = 'https://rdap.org/'
|
||||
|
||||
# Risk Scoring Weights
|
||||
RISK_WEIGHT_DOMAIN_AGE = 0.30
|
||||
RISK_WEIGHT_REPUTATION = 0.30
|
||||
RISK_WEIGHT_SSL = 0.20
|
||||
RISK_WEIGHT_DNS = 0.10
|
||||
RISK_WEIGHT_WHOIS = 0.10
|
||||
|
||||
# Pagination
|
||||
DEFAULT_PAGE_SIZE = 50
|
||||
MAX_PAGE_SIZE = 100
|
||||
|
||||
# Timeouts (seconds)
|
||||
HTTP_TIMEOUT = 30
|
||||
DNS_TIMEOUT = 10
|
||||
SSL_TIMEOUT = 10
|
||||
|
||||
@staticmethod
|
||||
def init_app(app):
|
||||
"""Initialize application configuration"""
|
||||
# Create logs directory if it doesn't exist
|
||||
log_dir = os.path.dirname(Config.LOG_FILE)
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
|
||||
|
||||
class DevelopmentConfig(Config):
|
||||
"""Development configuration"""
|
||||
DEBUG = True
|
||||
SQLALCHEMY_ECHO = False
|
||||
LOG_LEVEL = 'DEBUG'
|
||||
|
||||
|
||||
class ProductionConfig(Config):
|
||||
"""Production configuration"""
|
||||
DEBUG = False
|
||||
TESTING = False
|
||||
SQLALCHEMY_ECHO = False
|
||||
LOG_LEVEL = 'WARNING'
|
||||
|
||||
# Production security defaults (can be overridden by env vars)
|
||||
# If CORS_ORIGINS not set, default to empty (blocks all cross-origin)
|
||||
_cors_origins_env = os.getenv('CORS_ORIGINS', '')
|
||||
CORS_ORIGINS = [o.strip() for o in _cors_origins_env.split(',') if o.strip()] if _cors_origins_env else []
|
||||
|
||||
# Disable swagger by default in production (override with DISABLE_SWAGGER=false)
|
||||
DISABLE_SWAGGER = os.getenv('DISABLE_SWAGGER', 'true').lower() != 'false'
|
||||
|
||||
# Simple health by default in production
|
||||
SIMPLE_HEALTH = os.getenv('SIMPLE_HEALTH', 'true').lower() != 'false'
|
||||
|
||||
@classmethod
|
||||
def init_app(cls, app):
|
||||
Config.init_app(app)
|
||||
|
||||
# Production-specific initialization
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
|
||||
# Setup file handler with rotation
|
||||
file_handler = RotatingFileHandler(
|
||||
cls.LOG_FILE,
|
||||
maxBytes=10485760, # 10MB
|
||||
backupCount=10
|
||||
)
|
||||
file_handler.setLevel(logging.WARNING)
|
||||
file_handler.setFormatter(logging.Formatter(cls.LOG_FORMAT))
|
||||
app.logger.addHandler(file_handler)
|
||||
|
||||
|
||||
class TestingConfig(Config):
|
||||
"""Testing configuration"""
|
||||
TESTING = True
|
||||
DEBUG = True
|
||||
SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:'
|
||||
REDIS_URL = 'redis://localhost:6379/15' # Use separate Redis DB for testing
|
||||
WTF_CSRF_ENABLED = False
|
||||
|
||||
|
||||
# Configuration dictionary
|
||||
config: Dict[str, Any] = {
|
||||
'development': DevelopmentConfig,
|
||||
'production': ProductionConfig,
|
||||
'testing': TestingConfig,
|
||||
'default': DevelopmentConfig
|
||||
}
|
||||
|
||||
|
||||
def get_config(env: str = None) -> Config:
|
||||
"""Get configuration based on environment"""
|
||||
if env is None:
|
||||
env = os.getenv('FLASK_ENV', 'development')
|
||||
return config.get(env, config['default'])
|
||||
26
ai_platform/modules/domain_check/api/app/models/__init__.py
Normal file
26
ai_platform/modules/domain_check/api/app/models/__init__.py
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
"""
|
||||
SQLAlchemy Models Package
|
||||
"""
|
||||
from app.models.domain import Domain
|
||||
from app.models.whois import WhoisRecord
|
||||
from app.models.dns import DnsRecord
|
||||
from app.models.ssl import SslCertificate
|
||||
from app.models.reputation import ReputationScore
|
||||
from app.models.risk import RiskAssessment
|
||||
from app.models.check_history import CheckHistory
|
||||
from app.models.batch import BatchOperation
|
||||
from app.models.blacklist import Blacklist
|
||||
from app.models.api_usage import ApiUsageLog
|
||||
|
||||
__all__ = [
|
||||
'Domain',
|
||||
'WhoisRecord',
|
||||
'DnsRecord',
|
||||
'SslCertificate',
|
||||
'ReputationScore',
|
||||
'RiskAssessment',
|
||||
'CheckHistory',
|
||||
'BatchOperation',
|
||||
'Blacklist',
|
||||
'ApiUsageLog'
|
||||
]
|
||||
32
ai_platform/modules/domain_check/api/app/models/api_usage.py
Normal file
32
ai_platform/modules/domain_check/api/app/models/api_usage.py
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
"""API Usage Log Model"""
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from sqlalchemy.dialects.postgresql import UUID, JSONB
|
||||
from app import db
|
||||
|
||||
|
||||
class ApiUsageLog(db.Model):
|
||||
"""API usage logging model"""
|
||||
__tablename__ = 'api_usage_logs'
|
||||
|
||||
id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
service_name = db.Column(db.String(50), nullable=False, index=True)
|
||||
endpoint = db.Column(db.String(255))
|
||||
request_count = db.Column(db.Integer, default=1)
|
||||
response_time_ms = db.Column(db.Integer)
|
||||
status_code = db.Column(db.Integer, index=True)
|
||||
quota_used = db.Column(db.Integer)
|
||||
quota_remaining = db.Column(db.Integer)
|
||||
error_message = db.Column(db.Text)
|
||||
request_params = db.Column(JSONB)
|
||||
logged_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow, index=True)
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'id': str(self.id),
|
||||
'service_name': self.service_name,
|
||||
'endpoint': self.endpoint,
|
||||
'status_code': self.status_code,
|
||||
'response_time_ms': self.response_time_ms,
|
||||
'logged_at': self.logged_at.isoformat() if self.logged_at else None
|
||||
}
|
||||
39
ai_platform/modules/domain_check/api/app/models/batch.py
Normal file
39
ai_platform/modules/domain_check/api/app/models/batch.py
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
"""Batch Operation Model"""
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from sqlalchemy.dialects.postgresql import UUID, ARRAY, JSONB
|
||||
from app import db
|
||||
|
||||
|
||||
class BatchOperation(db.Model):
|
||||
"""Batch operation model"""
|
||||
__tablename__ = 'batch_operations'
|
||||
|
||||
id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
batch_id = db.Column(UUID(as_uuid=True), unique=True, nullable=False, index=True)
|
||||
total_domains = db.Column(db.Integer, nullable=False)
|
||||
completed_count = db.Column(db.Integer, default=0)
|
||||
failed_count = db.Column(db.Integer, default=0)
|
||||
status = db.Column(db.String(20), default='pending', index=True)
|
||||
priority = db.Column(db.String(20), default='normal', index=True)
|
||||
started_at = db.Column(db.DateTime(timezone=True))
|
||||
completed_at = db.Column(db.DateTime(timezone=True))
|
||||
estimated_completion_at = db.Column(db.DateTime(timezone=True))
|
||||
requested_by = db.Column(db.String(100))
|
||||
check_options = db.Column(JSONB)
|
||||
domain_list = db.Column(ARRAY(db.Text))
|
||||
error_log = db.Column(JSONB)
|
||||
summary = db.Column(JSONB)
|
||||
created_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow, index=True)
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'batch_id': str(self.batch_id),
|
||||
'total_domains': self.total_domains,
|
||||
'completed_count': self.completed_count,
|
||||
'failed_count': self.failed_count,
|
||||
'status': self.status,
|
||||
'progress_percentage': round((self.completed_count / self.total_domains * 100) if self.total_domains > 0 else 0, 2),
|
||||
'started_at': self.started_at.isoformat() if self.started_at else None,
|
||||
'completed_at': self.completed_at.isoformat() if self.completed_at else None
|
||||
}
|
||||
33
ai_platform/modules/domain_check/api/app/models/blacklist.py
Normal file
33
ai_platform/modules/domain_check/api/app/models/blacklist.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
"""Blacklist Model"""
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from sqlalchemy.dialects.postgresql import UUID, JSONB
|
||||
from app import db
|
||||
|
||||
|
||||
class Blacklist(db.Model):
|
||||
"""Blacklist model"""
|
||||
__tablename__ = 'blacklists'
|
||||
|
||||
id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
domain = db.Column(db.String(255), unique=True, nullable=False, index=True)
|
||||
reason = db.Column(db.Text)
|
||||
category = db.Column(db.String(50), index=True)
|
||||
source = db.Column(db.String(100))
|
||||
severity = db.Column(db.String(20), default='medium', index=True)
|
||||
added_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow)
|
||||
updated_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
is_active = db.Column(db.Boolean, default=True, index=True)
|
||||
expires_at = db.Column(db.DateTime(timezone=True))
|
||||
extra_data = db.Column(JSONB)
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'id': str(self.id),
|
||||
'domain': self.domain,
|
||||
'reason': self.reason,
|
||||
'category': self.category,
|
||||
'severity': self.severity,
|
||||
'is_active': self.is_active,
|
||||
'added_at': self.added_at.isoformat() if self.added_at else None
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
"""Check History Model"""
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from sqlalchemy.dialects.postgresql import UUID, JSONB
|
||||
from app import db
|
||||
|
||||
|
||||
class CheckHistory(db.Model):
|
||||
"""Check history model"""
|
||||
__tablename__ = 'check_history'
|
||||
|
||||
id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
check_id = db.Column(UUID(as_uuid=True), unique=True, nullable=False, index=True)
|
||||
domain_id = db.Column(UUID(as_uuid=True), db.ForeignKey('domains.id', ondelete='CASCADE'), nullable=False, index=True)
|
||||
risk_assessment_id = db.Column(UUID(as_uuid=True), db.ForeignKey('risk_assessments.id', ondelete='SET NULL'))
|
||||
requested_by = db.Column(db.String(100), default='api', index=True)
|
||||
request_ip = db.Column(db.String(45))
|
||||
user_agent = db.Column(db.Text)
|
||||
check_options = db.Column(JSONB)
|
||||
processing_time_ms = db.Column(db.Integer)
|
||||
cache_hit = db.Column(db.Boolean, default=False)
|
||||
changes_detected = db.Column(db.Boolean, default=False)
|
||||
change_summary = db.Column(JSONB)
|
||||
status = db.Column(db.String(20), default='completed', index=True)
|
||||
error_message = db.Column(db.Text)
|
||||
created_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow, index=True)
|
||||
|
||||
domain = db.relationship('Domain', back_populates='check_history')
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'id': str(self.id),
|
||||
'check_id': str(self.check_id),
|
||||
'status': self.status,
|
||||
'cache_hit': self.cache_hit,
|
||||
'processing_time_ms': self.processing_time_ms,
|
||||
'created_at': self.created_at.isoformat() if self.created_at else None
|
||||
}
|
||||
31
ai_platform/modules/domain_check/api/app/models/dns.py
Normal file
31
ai_platform/modules/domain_check/api/app/models/dns.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
"""DNS Record Model"""
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from app import db
|
||||
|
||||
|
||||
class DnsRecord(db.Model):
|
||||
"""DNS record model"""
|
||||
__tablename__ = 'dns_records'
|
||||
|
||||
id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
domain_id = db.Column(UUID(as_uuid=True), db.ForeignKey('domains.id', ondelete='CASCADE'), nullable=False, index=True)
|
||||
record_type = db.Column(db.String(10), nullable=False, index=True)
|
||||
record_value = db.Column(db.Text, nullable=False)
|
||||
ttl = db.Column(db.Integer)
|
||||
priority = db.Column(db.Integer)
|
||||
fetched_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow, index=True)
|
||||
created_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow)
|
||||
|
||||
domain = db.relationship('Domain', back_populates='dns_records')
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'id': str(self.id),
|
||||
'record_type': self.record_type,
|
||||
'record_value': self.record_value,
|
||||
'ttl': self.ttl,
|
||||
'priority': self.priority,
|
||||
'fetched_at': self.fetched_at.isoformat() if self.fetched_at else None
|
||||
}
|
||||
118
ai_platform/modules/domain_check/api/app/models/domain.py
Normal file
118
ai_platform/modules/domain_check/api/app/models/domain.py
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
"""
|
||||
Domain Model
|
||||
"""
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from sqlalchemy.dialects.postgresql import UUID, ARRAY
|
||||
from app import db
|
||||
|
||||
|
||||
class Domain(db.Model):
|
||||
"""Domain model - stores basic domain information"""
|
||||
|
||||
__tablename__ = 'domains'
|
||||
|
||||
id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
domain = db.Column(db.String(255), nullable=False, index=True)
|
||||
subdomain = db.Column(db.String(255), nullable=True)
|
||||
tld = db.Column(db.String(50), nullable=False, index=True)
|
||||
full_domain = db.Column(db.String(255), unique=True, nullable=False, index=True)
|
||||
|
||||
first_seen_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow)
|
||||
last_checked_at = db.Column(db.DateTime(timezone=True), nullable=True, index=True)
|
||||
check_count = db.Column(db.Integer, default=0)
|
||||
is_active = db.Column(db.Boolean, default=True, index=True)
|
||||
|
||||
created_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow)
|
||||
updated_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
# Relationships
|
||||
whois_records = db.relationship('WhoisRecord', back_populates='domain', lazy='dynamic', cascade='all, delete-orphan')
|
||||
dns_records = db.relationship('DnsRecord', back_populates='domain', lazy='dynamic', cascade='all, delete-orphan')
|
||||
ssl_certificates = db.relationship('SslCertificate', back_populates='domain', lazy='dynamic', cascade='all, delete-orphan')
|
||||
reputation_scores = db.relationship('ReputationScore', back_populates='domain', lazy='dynamic', cascade='all, delete-orphan')
|
||||
risk_assessments = db.relationship('RiskAssessment', back_populates='domain', lazy='dynamic', cascade='all, delete-orphan')
|
||||
check_history = db.relationship('CheckHistory', back_populates='domain', lazy='dynamic', cascade='all, delete-orphan')
|
||||
|
||||
def __repr__(self):
|
||||
return f'<Domain {self.full_domain}>'
|
||||
|
||||
def to_dict(self, include_relationships=False):
|
||||
"""Convert model to dictionary"""
|
||||
data = {
|
||||
'id': str(self.id),
|
||||
'domain': self.domain,
|
||||
'subdomain': self.subdomain,
|
||||
'tld': self.tld,
|
||||
'full_domain': self.full_domain,
|
||||
'first_seen_at': self.first_seen_at.isoformat() if self.first_seen_at else None,
|
||||
'last_checked_at': self.last_checked_at.isoformat() if self.last_checked_at else None,
|
||||
'check_count': self.check_count,
|
||||
'is_active': self.is_active,
|
||||
'created_at': self.created_at.isoformat() if self.created_at else None,
|
||||
'updated_at': self.updated_at.isoformat() if self.updated_at else None
|
||||
}
|
||||
|
||||
if include_relationships:
|
||||
# Get latest records
|
||||
latest_whois = self.whois_records.order_by(WhoisRecord.fetched_at.desc()).first()
|
||||
latest_risk = self.risk_assessments.order_by(RiskAssessment.assessed_at.desc()).first()
|
||||
|
||||
data['latest_whois'] = latest_whois.to_dict() if latest_whois else None
|
||||
data['latest_risk'] = latest_risk.to_dict() if latest_risk else None
|
||||
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def parse_domain(full_domain: str) -> dict:
|
||||
"""
|
||||
Parse a full domain into components
|
||||
|
||||
Args:
|
||||
full_domain: Full domain name (e.g., 'www.example.com')
|
||||
|
||||
Returns:
|
||||
dict with 'domain', 'subdomain', 'tld', 'full_domain'
|
||||
"""
|
||||
import tldextract
|
||||
|
||||
extracted = tldextract.extract(full_domain)
|
||||
|
||||
return {
|
||||
'domain': extracted.domain,
|
||||
'subdomain': extracted.subdomain if extracted.subdomain else None,
|
||||
'tld': extracted.suffix,
|
||||
'full_domain': full_domain.lower().strip()
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def get_or_create(cls, full_domain: str):
|
||||
"""
|
||||
Get existing domain or create new one
|
||||
|
||||
Args:
|
||||
full_domain: Full domain name
|
||||
|
||||
Returns:
|
||||
Tuple of (Domain instance, created boolean)
|
||||
"""
|
||||
domain = cls.query.filter_by(full_domain=full_domain.lower()).first()
|
||||
|
||||
if domain:
|
||||
return domain, False
|
||||
|
||||
# Parse domain components
|
||||
parsed = cls.parse_domain(full_domain)
|
||||
|
||||
# Create new domain
|
||||
domain = cls(
|
||||
domain=parsed['domain'],
|
||||
subdomain=parsed['subdomain'],
|
||||
tld=parsed['tld'],
|
||||
full_domain=parsed['full_domain']
|
||||
)
|
||||
|
||||
db.session.add(domain)
|
||||
db.session.commit()
|
||||
|
||||
return domain, True
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
"""Reputation Score Model"""
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from sqlalchemy.dialects.postgresql import UUID, ARRAY, JSONB
|
||||
from app import db
|
||||
|
||||
|
||||
class ReputationScore(db.Model):
|
||||
"""Reputation score from various sources"""
|
||||
__tablename__ = 'reputation_scores'
|
||||
|
||||
id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
domain_id = db.Column(UUID(as_uuid=True), db.ForeignKey('domains.id', ondelete='CASCADE'), nullable=False, index=True)
|
||||
source = db.Column(db.String(50), nullable=False, index=True)
|
||||
score = db.Column(db.Integer)
|
||||
malicious_count = db.Column(db.Integer, default=0)
|
||||
suspicious_count = db.Column(db.Integer, default=0)
|
||||
harmless_count = db.Column(db.Integer, default=0)
|
||||
undetected_count = db.Column(db.Integer, default=0)
|
||||
is_blacklisted = db.Column(db.Boolean, default=False, index=True)
|
||||
blacklist_names = db.Column(ARRAY(db.Text))
|
||||
is_typosquatting = db.Column(db.Boolean, default=False, index=True)
|
||||
typosquatting_target = db.Column(db.String(255))
|
||||
raw_response = db.Column(JSONB)
|
||||
checked_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow, index=True)
|
||||
created_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow)
|
||||
|
||||
domain = db.relationship('Domain', back_populates='reputation_scores')
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'id': str(self.id),
|
||||
'source': self.source,
|
||||
'score': self.score,
|
||||
'malicious_count': self.malicious_count,
|
||||
'suspicious_count': self.suspicious_count,
|
||||
'harmless_count': self.harmless_count,
|
||||
'is_blacklisted': self.is_blacklisted,
|
||||
'blacklist_names': self.blacklist_names,
|
||||
'is_typosquatting': self.is_typosquatting,
|
||||
'checked_at': self.checked_at.isoformat() if self.checked_at else None
|
||||
}
|
||||
43
ai_platform/modules/domain_check/api/app/models/risk.py
Normal file
43
ai_platform/modules/domain_check/api/app/models/risk.py
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
"""Risk Assessment Model"""
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from sqlalchemy.dialects.postgresql import UUID, JSONB
|
||||
from app import db
|
||||
|
||||
|
||||
class RiskAssessment(db.Model):
|
||||
"""Risk assessment model"""
|
||||
__tablename__ = 'risk_assessments'
|
||||
|
||||
id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
domain_id = db.Column(UUID(as_uuid=True), db.ForeignKey('domains.id', ondelete='CASCADE'), nullable=False, index=True)
|
||||
check_id = db.Column(UUID(as_uuid=True), unique=True, nullable=False, index=True)
|
||||
total_score = db.Column(db.Integer, nullable=False, index=True)
|
||||
risk_level = db.Column(db.String(20), nullable=False, index=True)
|
||||
domain_age_score = db.Column(db.Integer, default=0)
|
||||
domain_age_days = db.Column(db.Integer)
|
||||
ssl_score = db.Column(db.Integer, default=0)
|
||||
dns_score = db.Column(db.Integer, default=0)
|
||||
reputation_score = db.Column(db.Integer, default=0)
|
||||
whois_score = db.Column(db.Integer, default=0)
|
||||
factors = db.Column(JSONB)
|
||||
is_new_domain = db.Column(db.Boolean, default=False, index=True)
|
||||
is_suspicious = db.Column(db.Boolean, default=False, index=True)
|
||||
requires_manual_review = db.Column(db.Boolean, default=False)
|
||||
assessed_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow, index=True)
|
||||
created_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow)
|
||||
|
||||
domain = db.relationship('Domain', back_populates='risk_assessments')
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'id': str(self.id),
|
||||
'check_id': str(self.check_id),
|
||||
'total_score': self.total_score,
|
||||
'risk_level': self.risk_level,
|
||||
'domain_age_days': self.domain_age_days,
|
||||
'is_new_domain': self.is_new_domain,
|
||||
'is_suspicious': self.is_suspicious,
|
||||
'factors': self.factors,
|
||||
'assessed_at': self.assessed_at.isoformat() if self.assessed_at else None
|
||||
}
|
||||
40
ai_platform/modules/domain_check/api/app/models/ssl.py
Normal file
40
ai_platform/modules/domain_check/api/app/models/ssl.py
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
"""SSL Certificate Model"""
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from sqlalchemy.dialects.postgresql import UUID, JSONB
|
||||
from app import db
|
||||
|
||||
|
||||
class SslCertificate(db.Model):
|
||||
"""SSL Certificate model"""
|
||||
__tablename__ = 'ssl_certificates'
|
||||
|
||||
id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
domain_id = db.Column(UUID(as_uuid=True), db.ForeignKey('domains.id', ondelete='CASCADE'), nullable=False, index=True)
|
||||
issuer = db.Column(db.String(255), index=True)
|
||||
subject = db.Column(db.String(255))
|
||||
valid_from = db.Column(db.DateTime(timezone=True))
|
||||
valid_until = db.Column(db.DateTime(timezone=True), index=True)
|
||||
serial_number = db.Column(db.String(255))
|
||||
signature_algorithm = db.Column(db.String(100))
|
||||
key_size = db.Column(db.Integer)
|
||||
is_wildcard = db.Column(db.Boolean, default=False)
|
||||
is_self_signed = db.Column(db.Boolean, default=False)
|
||||
is_valid = db.Column(db.Boolean, default=True, index=True)
|
||||
certificate_chain = db.Column(JSONB)
|
||||
fetched_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow, index=True)
|
||||
created_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow)
|
||||
|
||||
domain = db.relationship('Domain', back_populates='ssl_certificates')
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'id': str(self.id),
|
||||
'issuer': self.issuer,
|
||||
'subject': self.subject,
|
||||
'valid_from': self.valid_from.isoformat() if self.valid_from else None,
|
||||
'valid_until': self.valid_until.isoformat() if self.valid_until else None,
|
||||
'is_valid': self.is_valid,
|
||||
'is_self_signed': self.is_self_signed,
|
||||
'days_until_expiry': (self.valid_until - datetime.utcnow()).days if self.valid_until and self.valid_until > datetime.utcnow() else 0
|
||||
}
|
||||
62
ai_platform/modules/domain_check/api/app/models/whois.py
Normal file
62
ai_platform/modules/domain_check/api/app/models/whois.py
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
"""
|
||||
WHOIS Record Model
|
||||
"""
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from sqlalchemy.dialects.postgresql import UUID, ARRAY, JSONB
|
||||
from app import db
|
||||
|
||||
|
||||
class WhoisRecord(db.Model):
|
||||
"""WHOIS/RDAP record model"""
|
||||
|
||||
__tablename__ = 'whois_records'
|
||||
|
||||
id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
domain_id = db.Column(UUID(as_uuid=True), db.ForeignKey('domains.id', ondelete='CASCADE'), nullable=False, index=True)
|
||||
|
||||
creation_date = db.Column(db.DateTime(timezone=True), index=True)
|
||||
expiration_date = db.Column(db.DateTime(timezone=True))
|
||||
updated_date = db.Column(db.DateTime(timezone=True))
|
||||
|
||||
registrar = db.Column(db.String(255), index=True)
|
||||
registrar_url = db.Column(db.String(500))
|
||||
registrant_org = db.Column(db.String(255))
|
||||
registrant_country = db.Column(db.String(2))
|
||||
admin_email = db.Column(db.String(255))
|
||||
|
||||
name_servers = db.Column(ARRAY(db.Text))
|
||||
status = db.Column(ARRAY(db.Text))
|
||||
dnssec = db.Column(db.Boolean)
|
||||
|
||||
raw_whois_data = db.Column(JSONB)
|
||||
data_source = db.Column(db.String(50), default='rdap', index=True)
|
||||
fetched_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow, index=True)
|
||||
created_at = db.Column(db.DateTime(timezone=True), default=datetime.utcnow)
|
||||
|
||||
# Relationship
|
||||
domain = db.relationship('Domain', back_populates='whois_records')
|
||||
|
||||
def __repr__(self):
|
||||
return f'<WhoisRecord domain_id={self.domain_id} source={self.data_source}>'
|
||||
|
||||
def to_dict(self):
|
||||
"""Convert to dictionary"""
|
||||
return {
|
||||
'id': str(self.id),
|
||||
'domain_id': str(self.domain_id),
|
||||
'creation_date': self.creation_date.isoformat() if self.creation_date else None,
|
||||
'expiration_date': self.expiration_date.isoformat() if self.expiration_date else None,
|
||||
'updated_date': self.updated_date.isoformat() if self.updated_date else None,
|
||||
'registrar': self.registrar,
|
||||
'registrar_url': self.registrar_url,
|
||||
'registrant_org': self.registrant_org,
|
||||
'registrant_country': self.registrant_country,
|
||||
'admin_email': self.admin_email,
|
||||
'name_servers': self.name_servers,
|
||||
'status': self.status,
|
||||
'dnssec': self.dnssec,
|
||||
'data_source': self.data_source,
|
||||
'fetched_at': self.fetched_at.isoformat() if self.fetched_at else None,
|
||||
'age_days': (datetime.utcnow() - self.creation_date).days if self.creation_date else None
|
||||
}
|
||||
14
ai_platform/modules/domain_check/api/app/routes/batch.py
Normal file
14
ai_platform/modules/domain_check/api/app/routes/batch.py
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
"""Batch routes - stub for now"""
|
||||
from flask import Blueprint, jsonify
|
||||
|
||||
batch_bp = Blueprint('batch', __name__)
|
||||
|
||||
@batch_bp.route('/check/batch', methods=['POST'])
|
||||
def batch_check():
|
||||
"""Batch domain check - TODO: implement"""
|
||||
return jsonify({'message': 'Not yet implemented'}), 501
|
||||
|
||||
@batch_bp.route('/batch/<string:batch_id>/status', methods=['GET'])
|
||||
def batch_status(batch_id):
|
||||
"""Get batch status - TODO: implement"""
|
||||
return jsonify({'message': 'Not yet implemented', 'batch_id': batch_id}), 501
|
||||
560
ai_platform/modules/domain_check/api/app/routes/check.py
Normal file
560
ai_platform/modules/domain_check/api/app/routes/check.py
Normal file
|
|
@ -0,0 +1,560 @@
|
|||
"""
|
||||
Domain Check Routes - Comprehensive Domain Verification API
|
||||
"""
|
||||
import uuid
|
||||
import logging
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime
|
||||
from flask import request, current_app
|
||||
from flask_restx import Namespace, Resource, fields
|
||||
from werkzeug.exceptions import BadRequest, HTTPException
|
||||
from app import db
|
||||
from app.models import Domain, WhoisRecord, DnsRecord, SslCertificate, RiskAssessment, CheckHistory
|
||||
from app.services.whois_service import WhoisService
|
||||
from app.services.risk_scorer import _coerce_datetime
|
||||
from app.services.dns_service import DNSService
|
||||
from app.services.ssl_service import SSLService
|
||||
from app.services.risk_scorer import RiskScorer
|
||||
from app.services.ip_intelligence_service import IPIntelligenceService
|
||||
from app.services.http_analysis_service import HTTPAnalysisService
|
||||
from app.services.blacklist_service import BlacklistService
|
||||
from app.services.subdomain_service import SubdomainService
|
||||
from app.services.port_scan_service import PortScanService
|
||||
from app.services.mail_intelligence_service import MailIntelligenceService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Create namespace
|
||||
api = Namespace('check', description='Domain checking operations')
|
||||
|
||||
# Import model definitions from api_models.py
|
||||
from app.api_models import create_api_models
|
||||
|
||||
# Create placeholder models
|
||||
_temp_api = Namespace('_temp')
|
||||
_models = create_api_models(_temp_api)
|
||||
|
||||
# Get the models we need for decorators
|
||||
check_request_model = _models.get('domain_check_request')
|
||||
check_response_model = _models.get('domain_check_response')
|
||||
error_response_model = _models.get('error_response')
|
||||
|
||||
|
||||
@api.route('/check')
|
||||
class DomainCheck(Resource):
|
||||
"""Domain Check Resource - Comprehensive domain verification"""
|
||||
|
||||
@api.doc(
|
||||
'check_domain',
|
||||
description='''Perform comprehensive domain verification including:
|
||||
|
||||
**Basic Checks:**
|
||||
- WHOIS lookup (domain age, registrar, status, name servers)
|
||||
- DNS record checking (A, AAAA, MX, TXT, NS, CNAME, SOA)
|
||||
- SSL certificate validation (validity, issuer, expiration)
|
||||
|
||||
**Advanced Checks (when enabled):**
|
||||
- IP Intelligence (geolocation, ASN, reverse DNS, hosting info)
|
||||
- HTTP Security Analysis (headers, technology detection)
|
||||
- Blacklist/Reputation checking (DNSBL, spam lists)
|
||||
- Port Scanning (open ports, dangerous services)
|
||||
- Subdomain Enumeration (Certificate Transparency)
|
||||
|
||||
Returns detailed domain information and comprehensive risk score (0-100).'''
|
||||
)
|
||||
@api.expect(check_request_model, validate=False)
|
||||
@api.response(200, 'Success - Domain check completed', check_response_model)
|
||||
@api.response(400, 'Bad Request - Invalid domain or parameters', error_response_model)
|
||||
@api.response(500, 'Internal Server Error - Check failed', error_response_model)
|
||||
def post(self):
|
||||
"""
|
||||
Check a single domain - Full comprehensive analysis
|
||||
"""
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
# Get request data
|
||||
try:
|
||||
data = api.payload or {}
|
||||
except BadRequest:
|
||||
data = {}
|
||||
|
||||
if not data or 'domain' not in data:
|
||||
return {
|
||||
'success': False,
|
||||
'error': {
|
||||
'code': 'INVALID_REQUEST',
|
||||
'message': 'Domain parameter is required',
|
||||
'status': 400
|
||||
}
|
||||
}, 400
|
||||
|
||||
domain_name = data['domain'].lower().strip()
|
||||
check_options = data.get('check_options', {
|
||||
'whois': True,
|
||||
'dns': True,
|
||||
'ssl': True,
|
||||
'ip_intelligence': True,
|
||||
'http_analysis': True,
|
||||
'blacklist': True,
|
||||
'port_scan': False, # Disabled by default (slow)
|
||||
'subdomains': False, # Disabled by default (slow)
|
||||
'force_refresh': False
|
||||
})
|
||||
|
||||
logger.info(f'Checking domain: {domain_name} with options: {check_options}')
|
||||
|
||||
# Generate check ID
|
||||
check_id = uuid.uuid4()
|
||||
|
||||
# Get or create domain
|
||||
domain, created = Domain.get_or_create(domain_name)
|
||||
|
||||
# Initialize all services
|
||||
whoxy_api_key = current_app.config.get('WHOXY_API_KEY')
|
||||
whois_service = WhoisService(whoxy_api_key=whoxy_api_key)
|
||||
dns_service = DNSService()
|
||||
ssl_service = SSLService()
|
||||
ip_service = IPIntelligenceService()
|
||||
http_service = HTTPAnalysisService()
|
||||
blacklist_service = BlacklistService()
|
||||
subdomain_service = SubdomainService()
|
||||
port_scan_service = PortScanService()
|
||||
mail_service = MailIntelligenceService()
|
||||
risk_scorer = RiskScorer()
|
||||
|
||||
# Collect all domain data for risk scoring
|
||||
domain_data = {
|
||||
'domain': domain_name,
|
||||
'whois': None,
|
||||
'dns': None,
|
||||
'ssl': None,
|
||||
'ip_intelligence': None,
|
||||
'http_analysis': None,
|
||||
'blacklist': None,
|
||||
'port_scan': None,
|
||||
'subdomains': None
|
||||
}
|
||||
|
||||
# Response data
|
||||
response_data = {
|
||||
'domain': domain_name,
|
||||
'check_id': str(check_id),
|
||||
'timestamp': datetime.utcnow().isoformat() + 'Z'
|
||||
}
|
||||
|
||||
# Primary IP for subsequent checks
|
||||
primary_ip = None
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Concurrent fetch. Network lookups are I/O-bound and independent,
|
||||
# so we run them in a thread pool instead of sequentially (cuts a
|
||||
# full check from ~sum-of-calls to ~max-of-calls). DB writes stay in
|
||||
# the main thread afterwards because the SQLAlchemy session is not
|
||||
# thread-safe. Each task runs inside an app context so services that
|
||||
# read current_app keep working off the main thread.
|
||||
# ----------------------------------------------------------------
|
||||
app_obj = current_app._get_current_object()
|
||||
|
||||
def _run(fn, *a, **kw):
|
||||
with app_obj.app_context():
|
||||
try:
|
||||
return fn(*a, **kw)
|
||||
except Exception as exc:
|
||||
logger.warning(f'{getattr(fn, "__name__", fn)} failed: {exc}')
|
||||
return None
|
||||
|
||||
# Phase 1 — tasks that only need the domain name.
|
||||
p1 = {}
|
||||
with ThreadPoolExecutor(max_workers=5) as ex:
|
||||
if check_options.get('whois', True):
|
||||
p1['whois'] = ex.submit(_run, whois_service.lookup, domain_name)
|
||||
if check_options.get('dns', True):
|
||||
p1['dns'] = ex.submit(_run, dns_service.lookup, domain_name)
|
||||
if check_options.get('ssl', True):
|
||||
p1['ssl'] = ex.submit(_run, ssl_service.check, domain_name)
|
||||
if check_options.get('http_analysis', True):
|
||||
p1['http'] = ex.submit(_run, http_service.analyze, domain_name)
|
||||
if check_options.get('subdomains', False):
|
||||
extra_subs = check_options.get('extra_subdomains') or []
|
||||
p1['subdomains'] = ex.submit(_run, subdomain_service.enumerate,
|
||||
get_root_domain(domain_name), extra_subs)
|
||||
|
||||
whois_data = p1['whois'].result() if 'whois' in p1 else None
|
||||
dns_data = p1['dns'].result() if 'dns' in p1 else None
|
||||
ssl_data = p1['ssl'].result() if 'ssl' in p1 else None
|
||||
http_data = p1['http'].result() if 'http' in p1 else None
|
||||
subdomain_data = p1['subdomains'].result() if 'subdomains' in p1 else None
|
||||
|
||||
if dns_data and dns_data.get('a_records'):
|
||||
primary_ip = dns_data['a_records'][0]
|
||||
|
||||
# Phase 2 — tasks that need DNS results (primary IP, MX/TXT).
|
||||
p2 = {}
|
||||
with ThreadPoolExecutor(max_workers=4) as ex:
|
||||
if check_options.get('ip_intelligence', True) and primary_ip:
|
||||
p2['ip'] = ex.submit(_run, ip_service.lookup, primary_ip)
|
||||
if check_options.get('blacklist', True):
|
||||
p2['blacklist'] = ex.submit(_run, blacklist_service.check_all, domain_name, primary_ip)
|
||||
if check_options.get('port_scan', False) and primary_ip:
|
||||
p2['port'] = ex.submit(_run, port_scan_service.quick_scan, primary_ip)
|
||||
if check_options.get('mail', True):
|
||||
mx = dns_data.get('mx_records') if dns_data else None
|
||||
txt = dns_data.get('txt_records') if dns_data else None
|
||||
p2['mail'] = ex.submit(_run, mail_service.analyze, domain_name, mx, txt,
|
||||
check_options.get('smtp_probe', False))
|
||||
|
||||
ip_data = p2['ip'].result() if 'ip' in p2 else None
|
||||
blacklist_data = p2['blacklist'].result() if 'blacklist' in p2 else None
|
||||
port_data = p2['port'].result() if 'port' in p2 else None
|
||||
mail_data = p2['mail'].result() if 'mail' in p2 else None
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Process results + persist (single-threaded, ordered).
|
||||
# ----------------------------------------------------------------
|
||||
|
||||
# 1. WHOIS
|
||||
if whois_data:
|
||||
domain_data['whois'] = whois_data
|
||||
response_data['whois'] = format_whois_response(whois_data)
|
||||
# Save WHOIS record
|
||||
try:
|
||||
whois_record = WhoisRecord(
|
||||
domain_id=domain.id,
|
||||
creation_date=whois_data.get('creation_date'),
|
||||
expiration_date=whois_data.get('expiration_date'),
|
||||
updated_date=whois_data.get('updated_date'),
|
||||
registrar=whois_data.get('registrar'),
|
||||
registrar_url=whois_data.get('registrar_url'),
|
||||
registrant_org=whois_data.get('registrant_org'),
|
||||
registrant_country=whois_data.get('registrant_country'),
|
||||
admin_email=whois_data.get('admin_email'),
|
||||
name_servers=whois_data.get('name_servers', []),
|
||||
status=whois_data.get('status', []),
|
||||
dnssec=whois_data.get('dnssec'),
|
||||
raw_whois_data=whois_data.get('raw_data', {}),
|
||||
data_source=whois_data.get('data_source', 'rdap')
|
||||
)
|
||||
db.session.add(whois_record)
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to save WHOIS record: {e}')
|
||||
|
||||
# 2. DNS
|
||||
if dns_data:
|
||||
domain_data['dns'] = dns_data
|
||||
response_data['dns'] = format_dns_response(dns_data)
|
||||
# Save DNS records
|
||||
try:
|
||||
for record_type in ['a_records', 'aaaa_records', 'mx_records', 'txt_records', 'ns_records', 'cname_records']:
|
||||
records = dns_data.get(record_type, [])
|
||||
if records:
|
||||
for record in records:
|
||||
if isinstance(record, dict): # MX records
|
||||
dns_record = DnsRecord(
|
||||
domain_id=domain.id,
|
||||
record_type='MX',
|
||||
record_value=record['host'],
|
||||
priority=record['priority']
|
||||
)
|
||||
else:
|
||||
dns_record = DnsRecord(
|
||||
domain_id=domain.id,
|
||||
record_type=record_type.replace('_records', '').upper(),
|
||||
record_value=str(record)
|
||||
)
|
||||
db.session.add(dns_record)
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to save DNS records: {e}')
|
||||
|
||||
# Registration / availability verdict (WHOIS + DNS combined signals)
|
||||
if check_options.get('whois', True) or check_options.get('dns', True):
|
||||
availability = determine_availability(whois_data, dns_data)
|
||||
response_data['availability'] = availability
|
||||
if 'whois' in response_data:
|
||||
response_data['whois']['is_registered'] = availability['is_registered']
|
||||
response_data['whois']['is_available'] = availability['is_available']
|
||||
|
||||
# 3. SSL Certificate
|
||||
if ssl_data:
|
||||
domain_data['ssl'] = ssl_data
|
||||
response_data['ssl'] = format_ssl_response(ssl_data)
|
||||
if ssl_data.get('has_ssl'):
|
||||
try:
|
||||
ssl_cert = SslCertificate(
|
||||
domain_id=domain.id,
|
||||
issuer=ssl_data.get('issuer'),
|
||||
subject=ssl_data.get('subject'),
|
||||
valid_from=ssl_data.get('valid_from'),
|
||||
valid_until=ssl_data.get('valid_until'),
|
||||
serial_number=ssl_data.get('serial_number'),
|
||||
signature_algorithm=ssl_data.get('signature_algorithm'),
|
||||
key_size=ssl_data.get('key_size'),
|
||||
is_wildcard=ssl_data.get('is_wildcard', False),
|
||||
is_self_signed=ssl_data.get('is_self_signed', False),
|
||||
is_valid=ssl_data.get('is_valid', True)
|
||||
)
|
||||
db.session.add(ssl_cert)
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to save SSL certificate: {e}')
|
||||
|
||||
# 4. IP Intelligence
|
||||
if ip_data:
|
||||
ip_data['hosting_score'] = ip_service.get_hosting_score(ip_data)
|
||||
domain_data['ip_intelligence'] = ip_data
|
||||
response_data['ip_intelligence'] = ip_data
|
||||
|
||||
# 5. HTTP Analysis
|
||||
if http_data:
|
||||
domain_data['http_analysis'] = http_data
|
||||
response_data['http_analysis'] = http_data
|
||||
|
||||
# 6. Blacklist
|
||||
if blacklist_data:
|
||||
domain_data['blacklist'] = blacklist_data
|
||||
response_data['blacklist'] = blacklist_data
|
||||
|
||||
# 7. Mail Intelligence (email infrastructure deep analysis)
|
||||
if mail_data:
|
||||
domain_data['mail'] = mail_data
|
||||
response_data['mail'] = mail_data
|
||||
|
||||
# 8. Port Scan (optional)
|
||||
if port_data:
|
||||
domain_data['port_scan'] = port_data
|
||||
response_data['port_scan'] = port_data
|
||||
|
||||
# 9. Subdomain Enumeration (optional)
|
||||
if subdomain_data:
|
||||
domain_data['subdomains'] = subdomain_data
|
||||
response_data['subdomains'] = subdomain_data
|
||||
|
||||
# 9. Calculate Risk Score
|
||||
logger.info(f'Calculating risk score for {domain_name}')
|
||||
risk_assessment_result = risk_scorer.calculate_risk(domain_data)
|
||||
|
||||
# Save risk assessment
|
||||
try:
|
||||
risk_assessment = RiskAssessment(
|
||||
domain_id=domain.id,
|
||||
check_id=check_id,
|
||||
total_score=risk_assessment_result['total_score'],
|
||||
risk_level=risk_assessment_result['risk_level'],
|
||||
domain_age_score=risk_assessment_result['individual_scores'].get('domain_age', 0),
|
||||
domain_age_days=next((f['details'].get('age_days') for f in risk_assessment_result['factors'] if f['factor'] == 'domain_age'), None),
|
||||
ssl_score=risk_assessment_result['individual_scores'].get('ssl', 0),
|
||||
dns_score=risk_assessment_result['individual_scores'].get('dns', 0),
|
||||
reputation_score=risk_assessment_result['individual_scores'].get('blacklist', 0),
|
||||
whois_score=risk_assessment_result['individual_scores'].get('whois', 0),
|
||||
factors=risk_assessment_result['factors'],
|
||||
is_new_domain=risk_assessment_result['is_new_domain'],
|
||||
is_suspicious=risk_assessment_result['is_suspicious'],
|
||||
requires_manual_review=risk_assessment_result['requires_manual_review']
|
||||
)
|
||||
db.session.add(risk_assessment)
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to save risk assessment: {e}')
|
||||
|
||||
# Save check history
|
||||
processing_time_ms = int((time.time() - start_time) * 1000)
|
||||
|
||||
try:
|
||||
check_history = CheckHistory(
|
||||
check_id=check_id,
|
||||
domain_id=domain.id,
|
||||
requested_by='api',
|
||||
request_ip=request.remote_addr,
|
||||
user_agent=request.headers.get('User-Agent'),
|
||||
check_options=check_options,
|
||||
processing_time_ms=processing_time_ms,
|
||||
cache_hit=False,
|
||||
status='completed'
|
||||
)
|
||||
db.session.add(check_history)
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to save check history: {e}')
|
||||
|
||||
# Commit all changes
|
||||
try:
|
||||
db.session.commit()
|
||||
except Exception as e:
|
||||
logger.error(f'Database commit failed: {e}')
|
||||
db.session.rollback()
|
||||
|
||||
# Build response
|
||||
response_data['risk_score'] = {
|
||||
'total': risk_assessment_result['total_score'],
|
||||
'level': risk_assessment_result['risk_level'],
|
||||
'factors': risk_assessment_result['factors'],
|
||||
'formula_breakdown': risk_assessment_result.get('formula_breakdown', []),
|
||||
'formula_string': risk_assessment_result.get('formula_string', ''),
|
||||
'thresholds': risk_assessment_result.get('risk_thresholds', {
|
||||
'low': '0-25',
|
||||
'medium': '26-50',
|
||||
'high': '51-75',
|
||||
'critical': '76-100'
|
||||
}),
|
||||
'is_new_domain': risk_assessment_result['is_new_domain'],
|
||||
'is_suspicious': risk_assessment_result['is_suspicious'],
|
||||
'is_blacklisted': risk_assessment_result.get('is_blacklisted', False),
|
||||
'requires_manual_review': risk_assessment_result['requires_manual_review']
|
||||
}
|
||||
|
||||
response = {
|
||||
'success': True,
|
||||
'data': response_data,
|
||||
'metadata': {
|
||||
'cached': False,
|
||||
'processing_time_ms': processing_time_ms,
|
||||
'api_version': current_app.config.get('API_VERSION', 'v1'),
|
||||
'checks_performed': [k for k, v in check_options.items() if v and k != 'force_refresh']
|
||||
}
|
||||
}
|
||||
|
||||
return response, 200
|
||||
|
||||
except HTTPException:
|
||||
# Re-raise HTTP exceptions (400, 404, etc.) as-is
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f'Error checking domain: {str(e)}', exc_info=True)
|
||||
db.session.rollback()
|
||||
|
||||
api.abort(500, f'An error occurred while checking the domain: {str(e)}',
|
||||
success=False,
|
||||
error={
|
||||
'code': 'INTERNAL_ERROR',
|
||||
'message': f'An error occurred while checking the domain: {str(e)}'
|
||||
})
|
||||
|
||||
|
||||
def get_root_domain(domain: str) -> str:
|
||||
"""Extract root domain from subdomain"""
|
||||
parts = domain.split('.')
|
||||
if len(parts) >= 2:
|
||||
# Handle common TLDs
|
||||
common_tlds = ['com', 'org', 'net', 'io', 'co', 'eu', 'ro', 'de', 'uk', 'fr']
|
||||
if parts[-1] in common_tlds:
|
||||
return '.'.join(parts[-2:])
|
||||
# Handle country code TLDs like .co.uk
|
||||
if len(parts) >= 3 and parts[-2] in ['co', 'com', 'org', 'net', 'gov']:
|
||||
return '.'.join(parts[-3:])
|
||||
return domain
|
||||
|
||||
|
||||
def _iso(value):
|
||||
"""Serialize a datetime to ISO-8601 Z, or None. Coerces stray strings safely."""
|
||||
dt = _coerce_datetime(value)
|
||||
return dt.isoformat() + 'Z' if dt else None
|
||||
|
||||
|
||||
def format_whois_response(whois_data: dict) -> dict:
|
||||
"""Format WHOIS data for API response"""
|
||||
creation_date = _coerce_datetime(whois_data.get('creation_date'))
|
||||
expiration_date = _coerce_datetime(whois_data.get('expiration_date'))
|
||||
|
||||
age_days = (datetime.utcnow() - creation_date).days if creation_date else None
|
||||
days_until_expiry = (expiration_date - datetime.utcnow()).days if expiration_date else None
|
||||
|
||||
return {
|
||||
'creation_date': _iso(whois_data.get('creation_date')),
|
||||
'expiration_date': _iso(whois_data.get('expiration_date')),
|
||||
'updated_date': _iso(whois_data.get('updated_date')),
|
||||
'registrar': whois_data.get('registrar'),
|
||||
'age_days': age_days,
|
||||
'days_until_expiry': days_until_expiry,
|
||||
'status': whois_data.get('status', []),
|
||||
'name_servers': whois_data.get('name_servers', []),
|
||||
'dnssec': whois_data.get('dnssec'),
|
||||
'registrant_org': whois_data.get('registrant_org'),
|
||||
'registrant_country': whois_data.get('registrant_country'),
|
||||
'is_registered': whois_data.get('is_registered'),
|
||||
'data_source': whois_data.get('data_source', 'whois')
|
||||
}
|
||||
|
||||
|
||||
def determine_availability(whois_data: dict, dns_data: dict) -> dict:
|
||||
"""Combine WHOIS and DNS signals into an explicit registration verdict.
|
||||
|
||||
WHOIS alone is unreliable for sparse registries (ROTLD .ro), so DNS
|
||||
resolution (NS/A records) is used as a strong corroborating signal.
|
||||
"""
|
||||
whois_signal = whois_data.get('is_registered') if whois_data else None
|
||||
|
||||
dns_has_records = False
|
||||
if dns_data:
|
||||
dns_has_records = bool(
|
||||
dns_data.get('ns_records') or dns_data.get('a_records') or
|
||||
dns_data.get('aaaa_records') or dns_data.get('mx_records')
|
||||
)
|
||||
|
||||
signals = {
|
||||
'whois_has_data': whois_signal is True,
|
||||
'dns_resolves': dns_has_records,
|
||||
}
|
||||
|
||||
# Decision: any positive signal => registered. Confidence is high when
|
||||
# WHOIS and DNS agree, low when relying on a single weak signal.
|
||||
if whois_signal is True or dns_has_records:
|
||||
is_registered = True
|
||||
confidence = 'high' if (whois_signal is True and dns_has_records) else 'medium'
|
||||
elif whois_signal is False:
|
||||
is_registered = False
|
||||
confidence = 'high' if not dns_has_records else 'low'
|
||||
else:
|
||||
# WHOIS inconclusive (None) and no DNS records => probably available.
|
||||
is_registered = False
|
||||
confidence = 'low'
|
||||
|
||||
return {
|
||||
'is_registered': is_registered,
|
||||
'is_available': not is_registered,
|
||||
'confidence': confidence,
|
||||
'signals': signals
|
||||
}
|
||||
|
||||
|
||||
def format_dns_response(dns_data: dict) -> dict:
|
||||
"""Format DNS data for API response"""
|
||||
return {
|
||||
'a_records': dns_data.get('a_records', []),
|
||||
'aaaa_records': dns_data.get('aaaa_records', []),
|
||||
'mx_records': dns_data.get('mx_records', []),
|
||||
'txt_records': dns_data.get('txt_records', []),
|
||||
'ns_records': dns_data.get('ns_records', []),
|
||||
'cname_records': dns_data.get('cname_records', []),
|
||||
'soa_record': dns_data.get('soa_record'),
|
||||
'has_spf': dns_data.get('has_spf', False),
|
||||
'has_dkim': dns_data.get('has_dkim', False),
|
||||
'has_dmarc': dns_data.get('has_dmarc', False),
|
||||
'spf_record': dns_data.get('spf_record'),
|
||||
'dmarc_record': dns_data.get('dmarc_record')
|
||||
}
|
||||
|
||||
|
||||
def format_ssl_response(ssl_data: dict) -> dict:
|
||||
"""Format SSL data for API response"""
|
||||
if not ssl_data.get('has_ssl'):
|
||||
return {
|
||||
'has_ssl': False,
|
||||
'error': ssl_data.get('error')
|
||||
}
|
||||
|
||||
valid_from = ssl_data.get('valid_from')
|
||||
valid_until = ssl_data.get('valid_until')
|
||||
|
||||
return {
|
||||
'has_ssl': True,
|
||||
'is_valid': ssl_data.get('is_valid', False),
|
||||
'is_self_signed': ssl_data.get('is_self_signed', False),
|
||||
'is_expired': ssl_data.get('is_expired', False),
|
||||
'is_wildcard': ssl_data.get('is_wildcard', False),
|
||||
'issuer': ssl_data.get('issuer'),
|
||||
'subject': ssl_data.get('subject'),
|
||||
'valid_from': valid_from.isoformat() + 'Z' if valid_from and hasattr(valid_from, 'isoformat') else str(valid_from) if valid_from else None,
|
||||
'valid_until': valid_until.isoformat() + 'Z' if valid_until and hasattr(valid_until, 'isoformat') else str(valid_until) if valid_until else None,
|
||||
'days_until_expiry': ssl_data.get('days_until_expiry'),
|
||||
'key_size': ssl_data.get('key_size'),
|
||||
'signature_algorithm': ssl_data.get('signature_algorithm'),
|
||||
'san': ssl_data.get('san', [])
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
"""Domain routes - stub for now"""
|
||||
from flask import Blueprint, jsonify
|
||||
|
||||
domain_bp = Blueprint('domain', __name__)
|
||||
|
||||
@domain_bp.route('/domain/<string:domain>', methods=['GET'])
|
||||
def get_domain(domain):
|
||||
"""Get domain details - TODO: implement"""
|
||||
return jsonify({'message': 'Not yet implemented', 'domain': domain}), 501
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
"""Search routes - stub for now"""
|
||||
from flask import Blueprint, jsonify
|
||||
|
||||
search_bp = Blueprint('search', __name__)
|
||||
|
||||
@search_bp.route('/search', methods=['GET'])
|
||||
def search_domains():
|
||||
"""Search domains - TODO: implement"""
|
||||
return jsonify({'message': 'Not yet implemented'}), 501
|
||||
9
ai_platform/modules/domain_check/api/app/routes/stats.py
Normal file
9
ai_platform/modules/domain_check/api/app/routes/stats.py
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
"""Stats routes - stub for now"""
|
||||
from flask import Blueprint, jsonify
|
||||
|
||||
stats_bp = Blueprint('stats', __name__)
|
||||
|
||||
@stats_bp.route('/stats', methods=['GET'])
|
||||
def get_stats():
|
||||
"""Get system statistics - TODO: implement"""
|
||||
return jsonify({'message': 'Not yet implemented'}), 501
|
||||
|
|
@ -0,0 +1,297 @@
|
|||
"""
|
||||
Blacklist Checking Service - DNSBL, Spam lists, Reputation checks
|
||||
"""
|
||||
import logging
|
||||
import socket
|
||||
import dns.resolver
|
||||
import requests
|
||||
from typing import Dict, List, Optional
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlacklistService:
|
||||
"""Service for checking IP/domain against various blacklists"""
|
||||
|
||||
def __init__(self):
|
||||
self.timeout = 5
|
||||
self.resolver = dns.resolver.Resolver()
|
||||
self.resolver.nameservers = ['8.8.8.8', '8.8.4.4']
|
||||
self.resolver.timeout = 3
|
||||
self.resolver.lifetime = 5
|
||||
|
||||
# DNSBL lists to check
|
||||
self.dnsbl_lists = [
|
||||
{'name': 'Spamhaus ZEN', 'zone': 'zen.spamhaus.org', 'type': 'spam'},
|
||||
{'name': 'SpamCop', 'zone': 'bl.spamcop.net', 'type': 'spam'},
|
||||
{'name': 'Barracuda', 'zone': 'b.barracudacentral.org', 'type': 'spam'},
|
||||
{'name': 'SORBS', 'zone': 'dnsbl.sorbs.net', 'type': 'spam'},
|
||||
{'name': 'URIBL', 'zone': 'multi.uribl.com', 'type': 'uri'},
|
||||
{'name': 'SURBL', 'zone': 'multi.surbl.org', 'type': 'uri'},
|
||||
{'name': 'Spamhaus DBL', 'zone': 'dbl.spamhaus.org', 'type': 'domain'},
|
||||
{'name': 'URIBL Black', 'zone': 'black.uribl.com', 'type': 'uri'}
|
||||
]
|
||||
|
||||
def check_ip(self, ip_address: str) -> Dict:
|
||||
"""
|
||||
Check IP address against multiple blacklists
|
||||
|
||||
Args:
|
||||
ip_address: IP address to check
|
||||
|
||||
Returns:
|
||||
Dictionary with blacklist check results
|
||||
"""
|
||||
result = {
|
||||
'ip': ip_address,
|
||||
'is_blacklisted': False,
|
||||
'blacklist_count': 0,
|
||||
'clean_count': 0,
|
||||
'total_checked': 0,
|
||||
'listings': [],
|
||||
'clean_lists': [],
|
||||
'check_errors': []
|
||||
}
|
||||
|
||||
# Reverse IP for DNSBL query
|
||||
reversed_ip = '.'.join(reversed(ip_address.split('.')))
|
||||
|
||||
# Check IP-based blacklists in parallel
|
||||
ip_lists = [bl for bl in self.dnsbl_lists if bl['type'] in ['spam']]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as executor:
|
||||
futures = {
|
||||
executor.submit(self._check_dnsbl, reversed_ip, bl): bl
|
||||
for bl in ip_lists
|
||||
}
|
||||
|
||||
for future in as_completed(futures, timeout=15):
|
||||
bl = futures[future]
|
||||
result['total_checked'] += 1
|
||||
|
||||
try:
|
||||
is_listed, response = future.result()
|
||||
if is_listed:
|
||||
result['is_blacklisted'] = True
|
||||
result['blacklist_count'] += 1
|
||||
result['listings'].append({
|
||||
'list_name': bl['name'],
|
||||
'list_zone': bl['zone'],
|
||||
'response': response,
|
||||
'type': bl['type']
|
||||
})
|
||||
else:
|
||||
result['clean_count'] += 1
|
||||
result['clean_lists'].append(bl['name'])
|
||||
except Exception as e:
|
||||
result['check_errors'].append({
|
||||
'list': bl['name'],
|
||||
'error': str(e)
|
||||
})
|
||||
|
||||
return result
|
||||
|
||||
def check_domain(self, domain: str) -> Dict:
|
||||
"""
|
||||
Check domain against domain-based blacklists
|
||||
|
||||
Args:
|
||||
domain: Domain name to check
|
||||
|
||||
Returns:
|
||||
Dictionary with blacklist check results
|
||||
"""
|
||||
result = {
|
||||
'domain': domain,
|
||||
'is_blacklisted': False,
|
||||
'blacklist_count': 0,
|
||||
'clean_count': 0,
|
||||
'total_checked': 0,
|
||||
'listings': [],
|
||||
'clean_lists': [],
|
||||
'check_errors': []
|
||||
}
|
||||
|
||||
# Check domain-based blacklists
|
||||
domain_lists = [bl for bl in self.dnsbl_lists if bl['type'] in ['domain', 'uri']]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as executor:
|
||||
futures = {
|
||||
executor.submit(self._check_domain_bl, domain, bl): bl
|
||||
for bl in domain_lists
|
||||
}
|
||||
|
||||
for future in as_completed(futures, timeout=15):
|
||||
bl = futures[future]
|
||||
result['total_checked'] += 1
|
||||
|
||||
try:
|
||||
is_listed, response = future.result()
|
||||
if is_listed:
|
||||
result['is_blacklisted'] = True
|
||||
result['blacklist_count'] += 1
|
||||
result['listings'].append({
|
||||
'list_name': bl['name'],
|
||||
'list_zone': bl['zone'],
|
||||
'response': response,
|
||||
'type': bl['type']
|
||||
})
|
||||
else:
|
||||
result['clean_count'] += 1
|
||||
result['clean_lists'].append(bl['name'])
|
||||
except Exception as e:
|
||||
result['check_errors'].append({
|
||||
'list': bl['name'],
|
||||
'error': str(e)
|
||||
})
|
||||
|
||||
return result
|
||||
|
||||
def check_all(self, domain: str, ip_address: str) -> Dict:
|
||||
"""
|
||||
Check both domain and IP against all blacklists
|
||||
|
||||
Returns:
|
||||
Combined blacklist check results
|
||||
"""
|
||||
result = {
|
||||
'domain': domain,
|
||||
'ip': ip_address,
|
||||
'is_blacklisted': False,
|
||||
'ip_blacklisted': False,
|
||||
'domain_blacklisted': False,
|
||||
'total_listings': 0,
|
||||
'ip_check': None,
|
||||
'domain_check': None,
|
||||
'reputation_score': 100, # Start with perfect score
|
||||
'risk_level': 'LOW'
|
||||
}
|
||||
|
||||
# Check IP
|
||||
if ip_address:
|
||||
ip_result = self.check_ip(ip_address)
|
||||
result['ip_check'] = ip_result
|
||||
result['ip_blacklisted'] = ip_result['is_blacklisted']
|
||||
result['total_listings'] += ip_result['blacklist_count']
|
||||
|
||||
# Check domain
|
||||
domain_result = self.check_domain(domain)
|
||||
result['domain_check'] = domain_result
|
||||
result['domain_blacklisted'] = domain_result['is_blacklisted']
|
||||
result['total_listings'] += domain_result['blacklist_count']
|
||||
|
||||
# Set overall blacklist status
|
||||
result['is_blacklisted'] = result['ip_blacklisted'] or result['domain_blacklisted']
|
||||
|
||||
# Calculate reputation score
|
||||
result['reputation_score'] = self._calculate_reputation_score(result)
|
||||
result['risk_level'] = self._get_risk_level(result['reputation_score'])
|
||||
|
||||
return result
|
||||
|
||||
def _check_dnsbl(self, reversed_ip: str, blacklist: Dict) -> tuple:
|
||||
"""Check reversed IP against a DNSBL"""
|
||||
try:
|
||||
query = f"{reversed_ip}.{blacklist['zone']}"
|
||||
answers = self.resolver.resolve(query, 'A')
|
||||
response = str(answers[0])
|
||||
|
||||
# 127.0.0.1 is an error code meaning "not authorized to query"
|
||||
# Real blacklist matches return 127.0.0.2, 127.0.0.4, etc.
|
||||
if response == '127.0.0.1':
|
||||
logger.debug(f"DNSBL {blacklist['name']} returned error code 127.0.0.1 (not authorized)")
|
||||
return False, None
|
||||
|
||||
# Valid blacklist match
|
||||
return True, response
|
||||
except dns.resolver.NXDOMAIN:
|
||||
# Not listed
|
||||
return False, None
|
||||
except dns.resolver.NoAnswer:
|
||||
return False, None
|
||||
except dns.resolver.Timeout:
|
||||
raise Exception('Timeout')
|
||||
except Exception as e:
|
||||
raise Exception(str(e))
|
||||
|
||||
def _check_domain_bl(self, domain: str, blacklist: Dict) -> tuple:
|
||||
"""Check domain against a domain blacklist"""
|
||||
try:
|
||||
query = f"{domain}.{blacklist['zone']}"
|
||||
answers = self.resolver.resolve(query, 'A')
|
||||
response = str(answers[0])
|
||||
|
||||
# 127.0.0.1 is an error code meaning "not authorized to query"
|
||||
# Real blacklist matches return 127.0.0.2, 127.0.0.4, etc.
|
||||
# This is common with URIBL when querying from unregistered resolvers
|
||||
if response == '127.0.0.1':
|
||||
logger.debug(f"Domain BL {blacklist['name']} returned error code 127.0.0.1 (not authorized)")
|
||||
return False, None
|
||||
|
||||
# Valid blacklist match
|
||||
return True, response
|
||||
except dns.resolver.NXDOMAIN:
|
||||
return False, None
|
||||
except dns.resolver.NoAnswer:
|
||||
return False, None
|
||||
except dns.resolver.Timeout:
|
||||
raise Exception('Timeout')
|
||||
except Exception as e:
|
||||
raise Exception(str(e))
|
||||
|
||||
def _calculate_reputation_score(self, result: Dict) -> int:
|
||||
"""
|
||||
Calculate reputation score (100 = perfect, 0 = worst)
|
||||
|
||||
Each blacklist listing reduces the score
|
||||
"""
|
||||
score = 100
|
||||
|
||||
# IP blacklist hits are more severe
|
||||
if result.get('ip_check'):
|
||||
ip_listings = result['ip_check'].get('blacklist_count', 0)
|
||||
score -= ip_listings * 25 # -25 per IP listing
|
||||
|
||||
# Domain blacklist hits
|
||||
if result.get('domain_check'):
|
||||
domain_listings = result['domain_check'].get('blacklist_count', 0)
|
||||
score -= domain_listings * 20 # -20 per domain listing
|
||||
|
||||
return max(0, min(100, score))
|
||||
|
||||
def _get_risk_level(self, score: int) -> str:
|
||||
"""Determine risk level from reputation score"""
|
||||
if score >= 80:
|
||||
return 'LOW'
|
||||
elif score >= 60:
|
||||
return 'MEDIUM'
|
||||
elif score >= 40:
|
||||
return 'HIGH'
|
||||
else:
|
||||
return 'CRITICAL'
|
||||
|
||||
def get_blacklist_score(self, blacklist_data: Dict) -> Dict:
|
||||
"""
|
||||
Get blacklist score for risk calculation (0 = good, 100 = bad)
|
||||
|
||||
This inverts the reputation score for consistency with other risk scores
|
||||
"""
|
||||
reputation = blacklist_data.get('reputation_score', 100)
|
||||
score = 100 - reputation # Invert: 100 reputation = 0 risk
|
||||
|
||||
reasons = []
|
||||
if blacklist_data.get('is_blacklisted'):
|
||||
if blacklist_data.get('ip_blacklisted'):
|
||||
reasons.append(f"IP on {blacklist_data['ip_check']['blacklist_count']} blacklist(s)")
|
||||
if blacklist_data.get('domain_blacklisted'):
|
||||
reasons.append(f"Domain on {blacklist_data['domain_check']['blacklist_count']} blacklist(s)")
|
||||
else:
|
||||
reasons.append('Not on any blacklists')
|
||||
|
||||
return {
|
||||
'score': score,
|
||||
'reasons': reasons,
|
||||
'is_clean': score == 0,
|
||||
'is_blacklisted': blacklist_data.get('is_blacklisted', False)
|
||||
}
|
||||
163
ai_platform/modules/domain_check/api/app/services/dns_service.py
Normal file
163
ai_platform/modules/domain_check/api/app/services/dns_service.py
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
"""
|
||||
DNS Service - handles DNS record lookups
|
||||
"""
|
||||
import logging
|
||||
import dns.resolver
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DNSService:
|
||||
"""DNS lookup service class"""
|
||||
|
||||
def __init__(self, timeout: int = 10):
|
||||
self.timeout = timeout
|
||||
self.resolver = dns.resolver.Resolver()
|
||||
self.resolver.timeout = timeout
|
||||
self.resolver.lifetime = timeout
|
||||
# Use public DNS servers (Google DNS) for reliability
|
||||
self.resolver.nameservers = ['8.8.8.8', '8.8.4.4']
|
||||
|
||||
def lookup(self, domain: str) -> Optional[Dict]:
|
||||
"""
|
||||
Perform comprehensive DNS lookup
|
||||
|
||||
Args:
|
||||
domain: Domain name to lookup
|
||||
|
||||
Returns:
|
||||
Dictionary with DNS records or None
|
||||
"""
|
||||
try:
|
||||
logger.info(f'DNS lookup for: {domain}')
|
||||
|
||||
dns_data = {
|
||||
'domain': domain,
|
||||
'a_records': self._get_a_records(domain),
|
||||
'aaaa_records': self._get_aaaa_records(domain),
|
||||
'mx_records': self._get_mx_records(domain),
|
||||
'txt_records': self._get_txt_records(domain),
|
||||
'ns_records': self._get_ns_records(domain),
|
||||
'cname_records': self._get_cname_records(domain),
|
||||
'soa_record': self._get_soa_record(domain),
|
||||
'has_a_records': False,
|
||||
'has_mx_records': False,
|
||||
'has_txt_records': False,
|
||||
'has_spf': False,
|
||||
'has_dkim': False,
|
||||
'has_dmarc': False
|
||||
}
|
||||
|
||||
# Set boolean flags
|
||||
dns_data['has_a_records'] = len(dns_data['a_records']) > 0
|
||||
dns_data['has_mx_records'] = len(dns_data['mx_records']) > 0
|
||||
dns_data['has_txt_records'] = len(dns_data['txt_records']) > 0
|
||||
|
||||
# Check for email security records
|
||||
for txt in dns_data['txt_records']:
|
||||
if txt.startswith('v=spf1'):
|
||||
dns_data['has_spf'] = True
|
||||
if 'dkim' in txt.lower():
|
||||
dns_data['has_dkim'] = True
|
||||
if 'v=DMARC1' in txt:
|
||||
dns_data['has_dmarc'] = True
|
||||
|
||||
return dns_data
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'DNS lookup failed for {domain}: {str(e)}')
|
||||
return None
|
||||
|
||||
def _get_a_records(self, domain: str) -> List[str]:
|
||||
"""Get A records (IPv4)"""
|
||||
try:
|
||||
answers = self.resolver.resolve(domain, 'A')
|
||||
return [str(rdata) for rdata in answers]
|
||||
except Exception as e:
|
||||
logger.debug(f'No A records for {domain}: {e}')
|
||||
return []
|
||||
|
||||
def _get_aaaa_records(self, domain: str) -> List[str]:
|
||||
"""Get AAAA records (IPv6)"""
|
||||
try:
|
||||
answers = self.resolver.resolve(domain, 'AAAA')
|
||||
return [str(rdata) for rdata in answers]
|
||||
except Exception as e:
|
||||
logger.debug(f'No AAAA records for {domain}: {e}')
|
||||
return []
|
||||
|
||||
def _get_mx_records(self, domain: str) -> List[Dict]:
|
||||
"""Get MX records (Mail servers)"""
|
||||
try:
|
||||
answers = self.resolver.resolve(domain, 'MX')
|
||||
return [
|
||||
{
|
||||
'priority': rdata.preference,
|
||||
'host': str(rdata.exchange).rstrip('.')
|
||||
}
|
||||
for rdata in answers
|
||||
]
|
||||
except Exception as e:
|
||||
logger.debug(f'No MX records for {domain}: {e}')
|
||||
return []
|
||||
|
||||
def _get_txt_records(self, domain: str) -> List[str]:
|
||||
"""Get TXT records"""
|
||||
try:
|
||||
answers = self.resolver.resolve(domain, 'TXT')
|
||||
records = []
|
||||
for rdata in answers:
|
||||
# TXT records can be split into multiple strings
|
||||
txt = ''.join([s.decode('utf-8') if isinstance(s, bytes) else str(s) for s in rdata.strings])
|
||||
records.append(txt)
|
||||
return records
|
||||
except Exception as e:
|
||||
logger.debug(f'No TXT records for {domain}: {e}')
|
||||
return []
|
||||
|
||||
def _get_ns_records(self, domain: str) -> List[str]:
|
||||
"""Get NS records (Name servers)"""
|
||||
try:
|
||||
answers = self.resolver.resolve(domain, 'NS')
|
||||
return [str(rdata).rstrip('.') for rdata in answers]
|
||||
except Exception as e:
|
||||
logger.debug(f'No NS records for {domain}: {e}')
|
||||
return []
|
||||
|
||||
def _get_cname_records(self, domain: str) -> List[str]:
|
||||
"""Get CNAME records"""
|
||||
try:
|
||||
answers = self.resolver.resolve(domain, 'CNAME')
|
||||
return [str(rdata).rstrip('.') for rdata in answers]
|
||||
except Exception as e:
|
||||
logger.debug(f'No CNAME records for {domain}: {e}')
|
||||
return []
|
||||
|
||||
def _get_soa_record(self, domain: str) -> Optional[Dict]:
|
||||
"""Get SOA record (Start of Authority)"""
|
||||
try:
|
||||
answers = self.resolver.resolve(domain, 'SOA')
|
||||
if answers:
|
||||
soa = answers[0]
|
||||
return {
|
||||
'mname': str(soa.mname).rstrip('.'),
|
||||
'rname': str(soa.rname).rstrip('.'),
|
||||
'serial': soa.serial,
|
||||
'refresh': soa.refresh,
|
||||
'retry': soa.retry,
|
||||
'expire': soa.expire,
|
||||
'minimum': soa.minimum
|
||||
}
|
||||
except Exception as e:
|
||||
logger.debug(f'No SOA record for {domain}: {e}')
|
||||
return None
|
||||
|
||||
def check_dnssec(self, domain: str) -> bool:
|
||||
"""Check if DNSSEC is enabled"""
|
||||
try:
|
||||
# Try to get DNSKEY records
|
||||
self.resolver.resolve(domain, 'DNSKEY')
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
|
@ -0,0 +1,369 @@
|
|||
"""
|
||||
HTTP Analysis Service - Headers, Technology Detection, Security Headers
|
||||
"""
|
||||
import logging
|
||||
import re
|
||||
import requests
|
||||
from typing import Dict, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HTTPAnalysisService:
|
||||
"""Service for analyzing HTTP responses and detecting technologies"""
|
||||
|
||||
def __init__(self):
|
||||
self.timeout = 15
|
||||
self.user_agent = 'Mozilla/5.0 (compatible; DomainCheck/1.0; +https://domain-check.local)'
|
||||
|
||||
def analyze(self, domain: str) -> Dict:
|
||||
"""
|
||||
Perform comprehensive HTTP analysis
|
||||
|
||||
Args:
|
||||
domain: Domain name to analyze
|
||||
|
||||
Returns:
|
||||
Dictionary with HTTP analysis results
|
||||
"""
|
||||
result = {
|
||||
'domain': domain,
|
||||
'http_status': None,
|
||||
'https_status': None,
|
||||
'has_https': False,
|
||||
'http_to_https_redirect': False,
|
||||
'final_url': None,
|
||||
'redirect_chain': [],
|
||||
'response_time_ms': None,
|
||||
'server': None,
|
||||
'powered_by': None,
|
||||
'security_headers': {},
|
||||
'missing_security_headers': [],
|
||||
'cookies': [],
|
||||
'technologies': [],
|
||||
'cms': None,
|
||||
'frameworks': [],
|
||||
'has_robots_txt': False,
|
||||
'has_sitemap': False,
|
||||
'has_favicon': False,
|
||||
'error': None
|
||||
}
|
||||
|
||||
try:
|
||||
# Test HTTPS first
|
||||
https_result = self._check_url(f'https://{domain}')
|
||||
if https_result.get('success'):
|
||||
result['has_https'] = True
|
||||
result['https_status'] = https_result.get('status_code')
|
||||
result['final_url'] = https_result.get('final_url')
|
||||
result['redirect_chain'] = https_result.get('redirect_chain', [])
|
||||
result['response_time_ms'] = https_result.get('response_time_ms')
|
||||
result['server'] = https_result.get('server')
|
||||
result['powered_by'] = https_result.get('powered_by')
|
||||
result['security_headers'] = https_result.get('security_headers', {})
|
||||
result['missing_security_headers'] = https_result.get('missing_security_headers', [])
|
||||
result['cookies'] = https_result.get('cookies', [])
|
||||
|
||||
# Detect technologies from response
|
||||
if https_result.get('body'):
|
||||
tech = self._detect_technologies(https_result['body'], https_result.get('headers', {}))
|
||||
result['technologies'] = tech.get('technologies', [])
|
||||
result['cms'] = tech.get('cms')
|
||||
result['frameworks'] = tech.get('frameworks', [])
|
||||
|
||||
# Test HTTP
|
||||
http_result = self._check_url(f'http://{domain}')
|
||||
if http_result.get('success'):
|
||||
result['http_status'] = http_result.get('status_code')
|
||||
# Check if HTTP redirects to HTTPS
|
||||
if http_result.get('final_url', '').startswith('https://'):
|
||||
result['http_to_https_redirect'] = True
|
||||
|
||||
# Check for common files
|
||||
result['has_robots_txt'] = self._check_file_exists(f'https://{domain}/robots.txt')
|
||||
result['has_sitemap'] = self._check_file_exists(f'https://{domain}/sitemap.xml')
|
||||
result['has_favicon'] = self._check_file_exists(f'https://{domain}/favicon.ico')
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"HTTP analysis failed for {domain}: {e}")
|
||||
result['error'] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def _check_url(self, url: str) -> Dict:
|
||||
"""Check a URL and gather response data"""
|
||||
result = {
|
||||
'success': False,
|
||||
'url': url,
|
||||
'status_code': None,
|
||||
'final_url': None,
|
||||
'redirect_chain': [],
|
||||
'response_time_ms': None,
|
||||
'server': None,
|
||||
'powered_by': None,
|
||||
'security_headers': {},
|
||||
'missing_security_headers': [],
|
||||
'cookies': [],
|
||||
'headers': {},
|
||||
'body': None
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.get(
|
||||
url,
|
||||
timeout=self.timeout,
|
||||
allow_redirects=True,
|
||||
headers={'User-Agent': self.user_agent},
|
||||
verify=True
|
||||
)
|
||||
|
||||
result['success'] = True
|
||||
result['status_code'] = response.status_code
|
||||
result['final_url'] = response.url
|
||||
result['response_time_ms'] = int(response.elapsed.total_seconds() * 1000)
|
||||
result['headers'] = dict(response.headers)
|
||||
|
||||
# Get redirect chain
|
||||
if response.history:
|
||||
result['redirect_chain'] = [
|
||||
{'url': r.url, 'status': r.status_code}
|
||||
for r in response.history
|
||||
]
|
||||
|
||||
# Extract server info
|
||||
result['server'] = response.headers.get('Server')
|
||||
result['powered_by'] = response.headers.get('X-Powered-By')
|
||||
|
||||
# Analyze security headers
|
||||
result['security_headers'], result['missing_security_headers'] = \
|
||||
self._analyze_security_headers(response.headers)
|
||||
|
||||
# Analyze cookies
|
||||
result['cookies'] = self._analyze_cookies(response.cookies)
|
||||
|
||||
# Get body for technology detection
|
||||
result['body'] = response.text[:50000] # Limit body size
|
||||
|
||||
except requests.exceptions.SSLError as e:
|
||||
result['error'] = f'SSL Error: {str(e)}'
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
result['error'] = f'Connection Error: {str(e)}'
|
||||
except requests.exceptions.Timeout:
|
||||
result['error'] = 'Timeout'
|
||||
except Exception as e:
|
||||
result['error'] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def _analyze_security_headers(self, headers) -> tuple:
|
||||
"""Analyze security headers"""
|
||||
security_headers = {}
|
||||
missing = []
|
||||
|
||||
# Define required security headers
|
||||
required_headers = {
|
||||
'Strict-Transport-Security': 'HSTS - Forces HTTPS',
|
||||
'X-Frame-Options': 'Prevents clickjacking',
|
||||
'X-Content-Type-Options': 'Prevents MIME sniffing',
|
||||
'X-XSS-Protection': 'XSS filtering (legacy)',
|
||||
'Content-Security-Policy': 'CSP - Controls resource loading',
|
||||
'Referrer-Policy': 'Controls referrer information',
|
||||
'Permissions-Policy': 'Controls browser features'
|
||||
}
|
||||
|
||||
for header, description in required_headers.items():
|
||||
value = headers.get(header)
|
||||
if value:
|
||||
security_headers[header] = {
|
||||
'value': value,
|
||||
'description': description,
|
||||
'present': True
|
||||
}
|
||||
else:
|
||||
missing.append({
|
||||
'header': header,
|
||||
'description': description,
|
||||
'severity': self._get_header_severity(header)
|
||||
})
|
||||
|
||||
return security_headers, missing
|
||||
|
||||
def _get_header_severity(self, header: str) -> str:
|
||||
"""Get severity level for missing header"""
|
||||
critical = ['Strict-Transport-Security', 'Content-Security-Policy']
|
||||
high = ['X-Frame-Options', 'X-Content-Type-Options']
|
||||
|
||||
if header in critical:
|
||||
return 'CRITICAL'
|
||||
elif header in high:
|
||||
return 'HIGH'
|
||||
return 'MEDIUM'
|
||||
|
||||
def _analyze_cookies(self, cookies) -> List[Dict]:
|
||||
"""Analyze cookies for security attributes"""
|
||||
analyzed = []
|
||||
|
||||
for cookie in cookies:
|
||||
cookie_info = {
|
||||
'name': cookie.name,
|
||||
'secure': cookie.secure,
|
||||
'httponly': cookie.has_nonstandard_attr('HttpOnly'),
|
||||
'samesite': cookie.get_nonstandard_attr('SameSite'),
|
||||
'issues': []
|
||||
}
|
||||
|
||||
# Check for security issues
|
||||
if not cookie.secure:
|
||||
cookie_info['issues'].append('Missing Secure flag')
|
||||
if not cookie_info['httponly']:
|
||||
cookie_info['issues'].append('Missing HttpOnly flag')
|
||||
if not cookie_info['samesite']:
|
||||
cookie_info['issues'].append('Missing SameSite attribute')
|
||||
|
||||
analyzed.append(cookie_info)
|
||||
|
||||
return analyzed
|
||||
|
||||
def _detect_technologies(self, body: str, headers: Dict) -> Dict:
|
||||
"""Detect technologies from response body and headers"""
|
||||
result = {
|
||||
'technologies': [],
|
||||
'cms': None,
|
||||
'frameworks': []
|
||||
}
|
||||
|
||||
body_lower = body.lower()
|
||||
|
||||
# CMS Detection
|
||||
cms_patterns = {
|
||||
'WordPress': [
|
||||
'wp-content', 'wp-includes', 'wordpress',
|
||||
'<meta name="generator" content="WordPress'
|
||||
],
|
||||
'Joomla': ['joomla', '/media/jui/', '/components/com_'],
|
||||
'Drupal': ['drupal', '/sites/default/files/', 'Drupal.settings'],
|
||||
'Magento': ['magento', 'mage/', '/skin/frontend/'],
|
||||
'Shopify': ['shopify', 'cdn.shopify.com'],
|
||||
'Wix': ['wix.com', '_wix_browser_sess'],
|
||||
'Squarespace': ['squarespace', 'static.squarespace.com']
|
||||
}
|
||||
|
||||
for cms, patterns in cms_patterns.items():
|
||||
for pattern in patterns:
|
||||
if pattern.lower() in body_lower:
|
||||
result['cms'] = cms
|
||||
result['technologies'].append(cms)
|
||||
break
|
||||
if result['cms']:
|
||||
break
|
||||
|
||||
# Framework detection
|
||||
framework_patterns = {
|
||||
'React': ['react', '_reactRootContainer', 'data-reactroot'],
|
||||
'Vue.js': ['vue', 'data-v-', '__vue__'],
|
||||
'Angular': ['ng-', 'angular', 'ng-app', 'ng-controller'],
|
||||
'jQuery': ['jquery', 'jQuery'],
|
||||
'Bootstrap': ['bootstrap', 'class="container', 'class="row'],
|
||||
'Tailwind CSS': ['tailwind', 'class="flex', 'class="grid'],
|
||||
'Laravel': ['laravel', 'csrf-token'],
|
||||
'Django': ['csrfmiddlewaretoken', 'django'],
|
||||
'Express': ['express'],
|
||||
'Next.js': ['next.js', '__NEXT_DATA__', '_next/'],
|
||||
'Nuxt.js': ['nuxt', '__NUXT__']
|
||||
}
|
||||
|
||||
for framework, patterns in framework_patterns.items():
|
||||
for pattern in patterns:
|
||||
if pattern.lower() in body_lower:
|
||||
if framework not in result['frameworks']:
|
||||
result['frameworks'].append(framework)
|
||||
result['technologies'].append(framework)
|
||||
break
|
||||
|
||||
# Additional technologies from meta tags
|
||||
generator_match = re.search(r'<meta[^>]+name=["\']generator["\'][^>]+content=["\']([^"\']+)["\']', body, re.I)
|
||||
if generator_match:
|
||||
generator = generator_match.group(1)
|
||||
result['technologies'].append(f'Generator: {generator}')
|
||||
|
||||
# Server-side detection from headers
|
||||
server = headers.get('Server', '').lower()
|
||||
if 'nginx' in server:
|
||||
result['technologies'].append('Nginx')
|
||||
elif 'apache' in server:
|
||||
result['technologies'].append('Apache')
|
||||
elif 'iis' in server:
|
||||
result['technologies'].append('Microsoft IIS')
|
||||
|
||||
powered_by = headers.get('X-Powered-By', '').lower()
|
||||
if 'php' in powered_by:
|
||||
result['technologies'].append('PHP')
|
||||
elif 'asp.net' in powered_by:
|
||||
result['technologies'].append('ASP.NET')
|
||||
|
||||
return result
|
||||
|
||||
def _check_file_exists(self, url: str) -> bool:
|
||||
"""Check if a file exists at URL"""
|
||||
try:
|
||||
response = requests.head(
|
||||
url,
|
||||
timeout=5,
|
||||
allow_redirects=True,
|
||||
headers={'User-Agent': self.user_agent}
|
||||
)
|
||||
return response.status_code == 200
|
||||
except:
|
||||
return False
|
||||
|
||||
def get_security_score(self, http_data: Dict) -> Dict:
|
||||
"""
|
||||
Calculate HTTP security score
|
||||
|
||||
Returns:
|
||||
Dict with score (0-100) and reasons
|
||||
"""
|
||||
score = 0
|
||||
reasons = []
|
||||
|
||||
# No HTTPS = very bad
|
||||
if not http_data.get('has_https'):
|
||||
score += 40
|
||||
reasons.append('No HTTPS support')
|
||||
|
||||
# No HTTP to HTTPS redirect
|
||||
if http_data.get('has_https') and not http_data.get('http_to_https_redirect'):
|
||||
score += 15
|
||||
reasons.append('No HTTP to HTTPS redirect')
|
||||
|
||||
# Missing security headers
|
||||
missing_headers = http_data.get('missing_security_headers', [])
|
||||
for header in missing_headers:
|
||||
if header.get('severity') == 'CRITICAL':
|
||||
score += 15
|
||||
reasons.append(f"Missing: {header['header']}")
|
||||
elif header.get('severity') == 'HIGH':
|
||||
score += 10
|
||||
reasons.append(f"Missing: {header['header']}")
|
||||
else:
|
||||
score += 5
|
||||
|
||||
# Cookie issues
|
||||
cookies = http_data.get('cookies', [])
|
||||
for cookie in cookies:
|
||||
if cookie.get('issues'):
|
||||
score += 5 * len(cookie['issues'])
|
||||
for issue in cookie['issues']:
|
||||
reasons.append(f"Cookie '{cookie['name']}': {issue}")
|
||||
|
||||
if score == 0:
|
||||
reasons.append('Good HTTP security configuration')
|
||||
|
||||
return {
|
||||
'score': min(100, score),
|
||||
'reasons': reasons[:10], # Limit reasons
|
||||
'is_secure': score <= 20,
|
||||
'needs_improvement': score > 20 and score <= 50,
|
||||
'is_insecure': score > 50
|
||||
}
|
||||
|
|
@ -0,0 +1,233 @@
|
|||
"""
|
||||
IP Intelligence Service - Geolocation, ASN, Reverse DNS, Hosting Info
|
||||
"""
|
||||
import logging
|
||||
import socket
|
||||
import requests
|
||||
from typing import Dict, Optional
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class IPIntelligenceService:
|
||||
"""Service for gathering IP intelligence data"""
|
||||
|
||||
def __init__(self):
|
||||
self.timeout = 10
|
||||
self.ipinfo_url = "https://ipinfo.io/{ip}/json"
|
||||
|
||||
def lookup(self, ip_address: str) -> Dict:
|
||||
"""
|
||||
Get comprehensive IP intelligence
|
||||
|
||||
Args:
|
||||
ip_address: IP address to lookup
|
||||
|
||||
Returns:
|
||||
Dictionary with IP intelligence data
|
||||
"""
|
||||
result = {
|
||||
'ip': ip_address,
|
||||
'reverse_dns': None,
|
||||
'geolocation': None,
|
||||
'asn': None,
|
||||
'isp': None,
|
||||
'organization': None,
|
||||
'is_datacenter': False,
|
||||
'is_residential': False,
|
||||
'hostname': None,
|
||||
'city': None,
|
||||
'region': None,
|
||||
'country': None,
|
||||
'country_code': None,
|
||||
'coordinates': None,
|
||||
'timezone': None,
|
||||
'data_source': 'ipinfo.io'
|
||||
}
|
||||
|
||||
try:
|
||||
# Run lookups in parallel
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
futures = {
|
||||
executor.submit(self._get_reverse_dns, ip_address): 'reverse_dns',
|
||||
executor.submit(self._get_ipinfo, ip_address): 'ipinfo'
|
||||
}
|
||||
|
||||
for future in as_completed(futures, timeout=15):
|
||||
lookup_type = futures[future]
|
||||
try:
|
||||
data = future.result()
|
||||
if lookup_type == 'reverse_dns':
|
||||
result['reverse_dns'] = data
|
||||
result['hostname'] = data
|
||||
elif lookup_type == 'ipinfo' and data:
|
||||
result.update(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"IP lookup {lookup_type} failed: {e}")
|
||||
|
||||
# Determine if datacenter or residential
|
||||
result['is_datacenter'] = self._is_datacenter(result)
|
||||
result['is_residential'] = not result['is_datacenter']
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"IP intelligence lookup failed for {ip_address}: {e}")
|
||||
result['error'] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def _get_reverse_dns(self, ip_address: str) -> Optional[str]:
|
||||
"""Get reverse DNS (PTR record)"""
|
||||
try:
|
||||
hostname, _, _ = socket.gethostbyaddr(ip_address)
|
||||
return hostname
|
||||
except (socket.herror, socket.gaierror):
|
||||
return None
|
||||
|
||||
def _get_ipinfo(self, ip_address: str) -> Optional[Dict]:
|
||||
"""Get IP info from ipinfo.io"""
|
||||
try:
|
||||
response = requests.get(
|
||||
self.ipinfo_url.format(ip=ip_address),
|
||||
timeout=self.timeout
|
||||
)
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
|
||||
# Parse ASN from org field (format: "AS8708 DIGI ROMANIA S.A.")
|
||||
org = data.get('org', '')
|
||||
asn = None
|
||||
isp = None
|
||||
if org:
|
||||
parts = org.split(' ', 1)
|
||||
if parts[0].startswith('AS'):
|
||||
asn = parts[0]
|
||||
isp = parts[1] if len(parts) > 1 else None
|
||||
else:
|
||||
isp = org
|
||||
|
||||
# Parse coordinates
|
||||
loc = data.get('loc', '')
|
||||
coordinates = None
|
||||
if loc:
|
||||
try:
|
||||
lat, lon = loc.split(',')
|
||||
coordinates = {
|
||||
'latitude': float(lat),
|
||||
'longitude': float(lon)
|
||||
}
|
||||
except:
|
||||
pass
|
||||
|
||||
return {
|
||||
'hostname': data.get('hostname'),
|
||||
'city': data.get('city'),
|
||||
'region': data.get('region'),
|
||||
'country': self._get_country_name(data.get('country')),
|
||||
'country_code': data.get('country'),
|
||||
'coordinates': coordinates,
|
||||
'timezone': data.get('timezone'),
|
||||
'asn': asn,
|
||||
'isp': isp,
|
||||
'organization': isp,
|
||||
'postal': data.get('postal')
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"ipinfo.io lookup failed: {e}")
|
||||
return None
|
||||
|
||||
def _is_datacenter(self, data: Dict) -> bool:
|
||||
"""Determine if IP is likely a datacenter/hosting IP"""
|
||||
indicators = []
|
||||
|
||||
# Check ISP/organization for hosting keywords
|
||||
org = (data.get('organization') or '').lower()
|
||||
isp = (data.get('isp') or '').lower()
|
||||
hostname = (data.get('hostname') or '').lower()
|
||||
|
||||
hosting_keywords = [
|
||||
'hosting', 'server', 'cloud', 'datacenter', 'data center',
|
||||
'hetzner', 'ovh', 'digitalocean', 'linode', 'vultr', 'aws',
|
||||
'amazon', 'google', 'microsoft', 'azure', 'contabo', 'hostinger',
|
||||
'godaddy', 'bluehost', 'namecheap', 'maghost', 'simpliq', 'host365'
|
||||
]
|
||||
|
||||
for keyword in hosting_keywords:
|
||||
if keyword in org or keyword in isp:
|
||||
return True
|
||||
|
||||
# Check hostname patterns
|
||||
if hostname:
|
||||
hosting_patterns = ['static', 'vps', 'server', 'host', 'cloud', 'dedicated']
|
||||
for pattern in hosting_patterns:
|
||||
if pattern in hostname:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _get_country_name(self, code: str) -> str:
|
||||
"""Convert country code to name"""
|
||||
countries = {
|
||||
'RO': 'Romania',
|
||||
'US': 'United States',
|
||||
'GB': 'United Kingdom',
|
||||
'DE': 'Germany',
|
||||
'FR': 'France',
|
||||
'NL': 'Netherlands',
|
||||
'UA': 'Ukraine',
|
||||
'RU': 'Russia',
|
||||
'CN': 'China',
|
||||
'IN': 'India',
|
||||
'JP': 'Japan',
|
||||
'BR': 'Brazil',
|
||||
'CA': 'Canada',
|
||||
'AU': 'Australia'
|
||||
}
|
||||
return countries.get(code, code)
|
||||
|
||||
def get_hosting_score(self, ip_data: Dict) -> Dict:
|
||||
"""
|
||||
Calculate hosting provider reputation score
|
||||
|
||||
Returns:
|
||||
Dict with score (0-100) and reasons
|
||||
"""
|
||||
score = 50 # Neutral baseline
|
||||
reasons = []
|
||||
|
||||
isp = (ip_data.get('isp') or '').lower()
|
||||
country_code = ip_data.get('country_code', '')
|
||||
|
||||
# Trusted hosting providers (lower score = better)
|
||||
trusted_hosts = ['google', 'amazon', 'microsoft', 'cloudflare', 'akamai']
|
||||
for host in trusted_hosts:
|
||||
if host in isp:
|
||||
score = 20
|
||||
reasons.append(f'Trusted provider: {isp}')
|
||||
break
|
||||
|
||||
# Romanian ISPs (neutral)
|
||||
ro_isps = ['digi', 'rcs', 'rds', 'telekom', 'orange', 'vodafone']
|
||||
for isp_name in ro_isps:
|
||||
if isp_name in isp:
|
||||
score = 40
|
||||
reasons.append(f'Romanian ISP: {isp}')
|
||||
break
|
||||
|
||||
# Check for reverse DNS mismatch (higher risk)
|
||||
if ip_data.get('is_datacenter') and not ip_data.get('reverse_dns'):
|
||||
score += 20
|
||||
reasons.append('Datacenter IP without reverse DNS')
|
||||
|
||||
# High-risk countries
|
||||
high_risk_countries = ['RU', 'CN', 'KP', 'IR', 'NG']
|
||||
if country_code in high_risk_countries:
|
||||
score += 30
|
||||
reasons.append(f'High-risk country: {country_code}')
|
||||
|
||||
return {
|
||||
'score': min(100, max(0, score)),
|
||||
'reasons': reasons,
|
||||
'is_trusted': score <= 30,
|
||||
'is_suspicious': score >= 70
|
||||
}
|
||||
|
|
@ -0,0 +1,374 @@
|
|||
"""
|
||||
Mail Intelligence Service - deep analysis of a domain's email infrastructure.
|
||||
|
||||
Covers what a plain MX/TXT dump does not:
|
||||
- SPF parsing (policy qualifier, include chain, DNS-lookup budget per RFC 7208)
|
||||
- DMARC policy (_dmarc) parsing (p/sp/pct/rua/aspf/adkim)
|
||||
- DKIM selector discovery (probes common selectors)
|
||||
- MX provider fingerprinting + STARTTLS reachability
|
||||
- MTA-STS (RFC 8461), TLS-RPT (RFC 8460), DANE/TLSA (RFC 7672)
|
||||
- Optional SMTP-level mailbox + catch-all probing (RCPT TO), degrades
|
||||
gracefully when outbound port 25 is blocked.
|
||||
|
||||
Everything is best-effort and never raises: a failed probe becomes a recorded
|
||||
'unknown'/False signal, so a single hiccup cannot 500 the parent request.
|
||||
"""
|
||||
import logging
|
||||
import smtplib
|
||||
import socket
|
||||
import ssl
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import dns.resolver
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Common DKIM selectors used by major providers / tooling.
|
||||
COMMON_DKIM_SELECTORS = [
|
||||
'default', 'google', 'selector1', 'selector2', 'k1', 'k2', 'dkim',
|
||||
'mail', 'smtp', 's1', 's2', 'mandrill', 'mailjet', 'sendgrid',
|
||||
'zoho', 'protonmail', 'protonmail2', 'fm1', 'fm2', 'fm3', 'mxvault',
|
||||
]
|
||||
|
||||
# MX hostname substrings -> human provider name.
|
||||
MX_PROVIDERS = [
|
||||
('google.com', 'Google Workspace'),
|
||||
('googlemail.com', 'Google Workspace'),
|
||||
('outlook.com', 'Microsoft 365'),
|
||||
('protection.outlook', 'Microsoft 365'),
|
||||
('zoho', 'Zoho Mail'),
|
||||
('protonmail', 'Proton Mail'),
|
||||
('proton.me', 'Proton Mail'),
|
||||
('mail.ru', 'Mail.ru'),
|
||||
('yandex', 'Yandex Mail'),
|
||||
('mimecast', 'Mimecast'),
|
||||
('pphosted', 'Proofpoint'),
|
||||
('messagelabs', 'Broadcom/Symantec'),
|
||||
('mailgun', 'Mailgun'),
|
||||
('sendgrid', 'SendGrid'),
|
||||
('amazonaws', 'Amazon SES/WorkMail'),
|
||||
('secureserver.net', 'GoDaddy'),
|
||||
('one.com', 'one.com'),
|
||||
('hostinger', 'Hostinger'),
|
||||
('gandi', 'Gandi'),
|
||||
('ovh', 'OVH'),
|
||||
('rotld', 'ROTLD'),
|
||||
]
|
||||
|
||||
|
||||
class MailIntelligenceService:
|
||||
"""Deep email-infrastructure analysis for a domain."""
|
||||
|
||||
def __init__(self, dns_timeout: int = 6, smtp_timeout: int = 8, http_timeout: int = 6):
|
||||
self.smtp_timeout = smtp_timeout
|
||||
self.http_timeout = http_timeout
|
||||
self.resolver = dns.resolver.Resolver()
|
||||
self.resolver.timeout = dns_timeout
|
||||
self.resolver.lifetime = dns_timeout
|
||||
self.resolver.nameservers = ['8.8.8.8', '1.1.1.1']
|
||||
|
||||
# -- public ------------------------------------------------------------
|
||||
|
||||
def analyze(self, domain: str, mx_records: Optional[List[Dict]] = None,
|
||||
txt_records: Optional[List[str]] = None,
|
||||
check_smtp: bool = False) -> Dict:
|
||||
domain = domain.lower().strip()
|
||||
if mx_records is None:
|
||||
mx_records = self._resolve_mx(domain)
|
||||
if txt_records is None:
|
||||
txt_records = self._resolve_txt(domain)
|
||||
|
||||
spf = self._analyze_spf(domain, txt_records)
|
||||
dmarc = self._analyze_dmarc(domain)
|
||||
dkim = self._discover_dkim(domain)
|
||||
mta_sts = self._analyze_mta_sts(domain)
|
||||
tls_rpt = self._analyze_tls_rpt(domain)
|
||||
mx = self._analyze_mx(mx_records, check_smtp=check_smtp)
|
||||
|
||||
result = {
|
||||
'domain': domain,
|
||||
'has_mail': bool(mx_records),
|
||||
'mx': mx,
|
||||
'spf': spf,
|
||||
'dmarc': dmarc,
|
||||
'dkim': dkim,
|
||||
'mta_sts': mta_sts,
|
||||
'tls_rpt': tls_rpt,
|
||||
}
|
||||
if check_smtp:
|
||||
result['deliverability'] = self._probe_deliverability(domain, mx_records)
|
||||
|
||||
result['provider'] = mx.get('provider') if mx else None
|
||||
result.update(self._grade(result))
|
||||
return result
|
||||
|
||||
# -- DNS helpers -------------------------------------------------------
|
||||
|
||||
def _resolve_mx(self, domain: str) -> List[Dict]:
|
||||
try:
|
||||
answers = self.resolver.resolve(domain, 'MX')
|
||||
return sorted(
|
||||
[{'priority': r.preference, 'host': str(r.exchange).rstrip('.')} for r in answers],
|
||||
key=lambda m: m['priority']
|
||||
)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def _resolve_txt(self, name: str) -> List[str]:
|
||||
try:
|
||||
answers = self.resolver.resolve(name, 'TXT')
|
||||
out = []
|
||||
for r in answers:
|
||||
out.append(''.join(
|
||||
s.decode('utf-8', 'ignore') if isinstance(s, bytes) else str(s)
|
||||
for s in r.strings
|
||||
))
|
||||
return out
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
# -- SPF ---------------------------------------------------------------
|
||||
|
||||
def _analyze_spf(self, domain: str, txt_records: List[str]) -> Dict:
|
||||
spf_record = next((t for t in (txt_records or []) if t.lower().startswith('v=spf1')), None)
|
||||
if not spf_record:
|
||||
return {'present': False, 'record': None, 'policy': None,
|
||||
'lookup_count': 0, 'includes': [], 'issues': ['No SPF record']}
|
||||
|
||||
tokens = spf_record.split()
|
||||
includes, lookup_terms, policy = [], 0, 'neutral'
|
||||
# Mechanisms that cost a DNS lookup (RFC 7208 §4.6.4, max 10).
|
||||
lookup_mechs = ('include:', 'a', 'mx', 'ptr', 'exists:', 'redirect=')
|
||||
for tok in tokens[1:]:
|
||||
low = tok.lower()
|
||||
if low.startswith('include:'):
|
||||
includes.append(tok.split(':', 1)[1])
|
||||
lookup_terms += 1
|
||||
elif low.startswith(lookup_mechs) or low in ('a', 'mx', 'ptr'):
|
||||
lookup_terms += 1
|
||||
if low.endswith('all'):
|
||||
qual = low[0] if low[0] in '-~?+' else '+'
|
||||
policy = {'-': 'fail (strict)', '~': 'softfail', '?': 'neutral',
|
||||
'+': 'pass (insecure +all)'}.get(qual, 'neutral')
|
||||
|
||||
issues = []
|
||||
if lookup_terms > 10:
|
||||
issues.append(f'Exceeds 10 DNS-lookup limit ({lookup_terms}) -> SPF permerror')
|
||||
if policy.startswith('pass'):
|
||||
issues.append('+all allows anyone to send as this domain')
|
||||
if policy == 'neutral':
|
||||
issues.append('?all provides no protection')
|
||||
return {'present': True, 'record': spf_record, 'policy': policy,
|
||||
'lookup_count': lookup_terms, 'includes': includes, 'issues': issues}
|
||||
|
||||
# -- DMARC -------------------------------------------------------------
|
||||
|
||||
def _analyze_dmarc(self, domain: str) -> Dict:
|
||||
records = self._resolve_txt(f'_dmarc.{domain}')
|
||||
rec = next((t for t in records if t.lower().startswith('v=dmarc1')), None)
|
||||
if not rec:
|
||||
return {'present': False, 'record': None, 'policy': None,
|
||||
'pct': None, 'rua': [], 'issues': ['No DMARC record']}
|
||||
tags = {}
|
||||
for part in rec.split(';'):
|
||||
if '=' in part:
|
||||
k, v = part.split('=', 1)
|
||||
tags[k.strip().lower()] = v.strip()
|
||||
policy = tags.get('p')
|
||||
issues = []
|
||||
if policy == 'none':
|
||||
issues.append('p=none is monitor-only, does not block spoofing')
|
||||
if not policy:
|
||||
issues.append('Missing p= tag')
|
||||
if not tags.get('rua'):
|
||||
issues.append('No aggregate reporting (rua) configured')
|
||||
return {'present': True, 'record': rec, 'policy': policy,
|
||||
'subdomain_policy': tags.get('sp'),
|
||||
'pct': tags.get('pct', '100'),
|
||||
'rua': [a.strip() for a in tags.get('rua', '').split(',') if a.strip()],
|
||||
'alignment': {'aspf': tags.get('aspf', 'r'), 'adkim': tags.get('adkim', 'r')},
|
||||
'issues': issues}
|
||||
|
||||
# -- DKIM --------------------------------------------------------------
|
||||
|
||||
def _discover_dkim(self, domain: str) -> Dict:
|
||||
found = []
|
||||
|
||||
def probe(sel):
|
||||
recs = self._resolve_txt(f'{sel}._domainkey.{domain}')
|
||||
for r in recs:
|
||||
if 'v=dkim1' in r.lower() or 'p=' in r:
|
||||
return sel
|
||||
return None
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as ex:
|
||||
futures = {ex.submit(probe, s): s for s in COMMON_DKIM_SELECTORS}
|
||||
for fut in as_completed(futures):
|
||||
try:
|
||||
sel = fut.result()
|
||||
if sel:
|
||||
found.append(sel)
|
||||
except Exception:
|
||||
pass
|
||||
return {'present': bool(found), 'selectors_found': sorted(found),
|
||||
'note': 'Probes common selectors only; absence is not proof of no DKIM'}
|
||||
|
||||
# -- MTA-STS / TLS-RPT / DANE ------------------------------------------
|
||||
|
||||
def _analyze_mta_sts(self, domain: str) -> Dict:
|
||||
txt = self._resolve_txt(f'_mta-sts.{domain}')
|
||||
has_dns = any('v=stsv1' in t.lower() for t in txt)
|
||||
policy = None
|
||||
mode = None
|
||||
if has_dns:
|
||||
try:
|
||||
resp = requests.get(
|
||||
f'https://mta-sts.{domain}/.well-known/mta-sts.txt',
|
||||
timeout=self.http_timeout, allow_redirects=False
|
||||
)
|
||||
if resp.status_code == 200 and 'version' in resp.text.lower():
|
||||
policy = resp.text[:2000]
|
||||
for line in resp.text.splitlines():
|
||||
if line.lower().startswith('mode:'):
|
||||
mode = line.split(':', 1)[1].strip()
|
||||
except Exception:
|
||||
pass
|
||||
return {'present': has_dns, 'mode': mode, 'policy_fetched': policy is not None}
|
||||
|
||||
def _analyze_tls_rpt(self, domain: str) -> Dict:
|
||||
txt = self._resolve_txt(f'_smtp._tls.{domain}')
|
||||
rec = next((t for t in txt if 'v=tlsrptv1' in t.lower()), None)
|
||||
return {'present': rec is not None, 'record': rec}
|
||||
|
||||
def _check_tlsa(self, mx_host: str) -> bool:
|
||||
try:
|
||||
self.resolver.resolve(f'_25._tcp.{mx_host}', 'TLSA')
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# -- MX analysis -------------------------------------------------------
|
||||
|
||||
def _provider_for(self, host: str) -> Optional[str]:
|
||||
h = host.lower()
|
||||
for needle, name in MX_PROVIDERS:
|
||||
if needle in h:
|
||||
return name
|
||||
return None
|
||||
|
||||
def _analyze_mx(self, mx_records: List[Dict], check_smtp: bool) -> Dict:
|
||||
if not mx_records:
|
||||
return {'count': 0, 'hosts': [], 'provider': None, 'starttls': None}
|
||||
|
||||
provider = None
|
||||
hosts_out = []
|
||||
|
||||
def inspect(mx):
|
||||
host = mx['host']
|
||||
info = {
|
||||
'host': host, 'priority': mx['priority'],
|
||||
'provider': self._provider_for(host),
|
||||
'addresses': self._resolve_addresses(host),
|
||||
'dane_tlsa': self._check_tlsa(host),
|
||||
}
|
||||
if check_smtp:
|
||||
info.update(self._smtp_starttls(host))
|
||||
return info
|
||||
|
||||
with ThreadPoolExecutor(max_workers=min(5, len(mx_records))) as ex:
|
||||
for info in ex.map(inspect, mx_records[:5]):
|
||||
hosts_out.append(info)
|
||||
if not provider and info.get('provider'):
|
||||
provider = info['provider']
|
||||
|
||||
starttls = None
|
||||
if check_smtp:
|
||||
tls_flags = [h.get('starttls') for h in hosts_out if 'starttls' in h]
|
||||
if tls_flags:
|
||||
starttls = all(tls_flags)
|
||||
return {'count': len(mx_records), 'hosts': hosts_out,
|
||||
'provider': provider, 'starttls': starttls,
|
||||
'dane': any(h.get('dane_tlsa') for h in hosts_out)}
|
||||
|
||||
def _resolve_addresses(self, host: str) -> List[str]:
|
||||
out = []
|
||||
for rtype in ('A', 'AAAA'):
|
||||
try:
|
||||
out += [str(r) for r in self.resolver.resolve(host, rtype)]
|
||||
except Exception:
|
||||
pass
|
||||
return out
|
||||
|
||||
def _smtp_starttls(self, host: str) -> Dict:
|
||||
"""Connect on :25 and check STARTTLS. Degrades gracefully if blocked."""
|
||||
try:
|
||||
with smtplib.SMTP(host, 25, timeout=self.smtp_timeout) as smtp:
|
||||
banner = smtp.ehlo()
|
||||
supports_tls = smtp.has_extn('starttls')
|
||||
tls_ok = False
|
||||
if supports_tls:
|
||||
try:
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
smtp.starttls(context=ctx)
|
||||
tls_ok = True
|
||||
except Exception:
|
||||
tls_ok = False
|
||||
return {'reachable': True, 'starttls': supports_tls,
|
||||
'starttls_negotiated': tls_ok,
|
||||
'banner_code': banner[0] if banner else None}
|
||||
except (socket.timeout, ConnectionRefusedError, OSError) as e:
|
||||
return {'reachable': False, 'starttls': False,
|
||||
'reason': f'port 25 unreachable from server ({type(e).__name__})'}
|
||||
|
||||
# -- optional deliverability probe ------------------------------------
|
||||
|
||||
def _probe_deliverability(self, domain: str, mx_records: List[Dict]) -> Dict:
|
||||
"""SMTP RCPT probe for catch-all detection. Best-effort; many networks
|
||||
block outbound :25 and many servers grey-list, so results are advisory."""
|
||||
if not mx_records:
|
||||
return {'tested': False, 'reason': 'no MX'}
|
||||
host = mx_records[0]['host']
|
||||
try:
|
||||
with smtplib.SMTP(host, 25, timeout=self.smtp_timeout) as smtp:
|
||||
smtp.ehlo()
|
||||
smtp.mail('probe@example.com')
|
||||
# Random-looking address: if accepted, server is catch-all.
|
||||
code_random, _ = smtp.rcpt(f'zz-no-such-user-9281@{domain}')
|
||||
catch_all = code_random in (250, 251)
|
||||
return {'tested': True, 'catch_all': catch_all,
|
||||
'rcpt_code': code_random,
|
||||
'note': 'Advisory only; greylisting/anti-harvesting can skew results'}
|
||||
except (socket.timeout, ConnectionRefusedError, OSError) as e:
|
||||
return {'tested': False, 'reason': f'port 25 blocked/unreachable ({type(e).__name__})'}
|
||||
except Exception as e:
|
||||
return {'tested': False, 'reason': str(e)}
|
||||
|
||||
# -- grading -----------------------------------------------------------
|
||||
|
||||
def _grade(self, r: Dict) -> Dict:
|
||||
"""0-100 email-security score (higher = better) + letter grade + summary."""
|
||||
score = 0
|
||||
if r['spf'].get('present'):
|
||||
score += 20
|
||||
if r['spf'].get('policy', '').startswith('fail'):
|
||||
score += 10
|
||||
if r['dmarc'].get('present'):
|
||||
score += 20
|
||||
if r['dmarc'].get('policy') in ('quarantine', 'reject'):
|
||||
score += 15
|
||||
if r['dkim'].get('present'):
|
||||
score += 15
|
||||
if r['mta_sts'].get('present'):
|
||||
score += 10
|
||||
if r['mta_sts'].get('mode') == 'enforce':
|
||||
score += 5
|
||||
if r['tls_rpt'].get('present'):
|
||||
score += 5
|
||||
score = min(score, 100)
|
||||
grade = ('A' if score >= 85 else 'B' if score >= 70 else
|
||||
'C' if score >= 50 else 'D' if score >= 30 else 'F')
|
||||
return {'mail_security_score': score, 'mail_security_grade': grade}
|
||||
|
|
@ -0,0 +1,265 @@
|
|||
"""
|
||||
Port Scanning Service - Detect open ports and services
|
||||
"""
|
||||
import logging
|
||||
import socket
|
||||
from typing import Dict, List, Optional
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PortScanService:
|
||||
"""Service for port scanning and service detection"""
|
||||
|
||||
def __init__(self):
|
||||
self.timeout = 3
|
||||
|
||||
# Common ports to scan with service info
|
||||
self.common_ports = {
|
||||
21: {'service': 'FTP', 'category': 'file_transfer', 'risk': 'medium'},
|
||||
22: {'service': 'SSH', 'category': 'remote_access', 'risk': 'low'},
|
||||
23: {'service': 'Telnet', 'category': 'remote_access', 'risk': 'critical'},
|
||||
25: {'service': 'SMTP', 'category': 'email', 'risk': 'low'},
|
||||
53: {'service': 'DNS', 'category': 'infrastructure', 'risk': 'low'},
|
||||
80: {'service': 'HTTP', 'category': 'web', 'risk': 'low'},
|
||||
110: {'service': 'POP3', 'category': 'email', 'risk': 'medium'},
|
||||
111: {'service': 'RPCBind', 'category': 'infrastructure', 'risk': 'high'},
|
||||
135: {'service': 'MS-RPC', 'category': 'windows', 'risk': 'high'},
|
||||
139: {'service': 'NetBIOS', 'category': 'windows', 'risk': 'high'},
|
||||
143: {'service': 'IMAP', 'category': 'email', 'risk': 'low'},
|
||||
443: {'service': 'HTTPS', 'category': 'web', 'risk': 'low'},
|
||||
445: {'service': 'SMB', 'category': 'windows', 'risk': 'critical'},
|
||||
465: {'service': 'SMTPS', 'category': 'email', 'risk': 'low'},
|
||||
587: {'service': 'SMTP Submission', 'category': 'email', 'risk': 'low'},
|
||||
993: {'service': 'IMAPS', 'category': 'email', 'risk': 'low'},
|
||||
995: {'service': 'POP3S', 'category': 'email', 'risk': 'low'},
|
||||
1433: {'service': 'MS-SQL', 'category': 'database', 'risk': 'critical'},
|
||||
1521: {'service': 'Oracle DB', 'category': 'database', 'risk': 'critical'},
|
||||
1723: {'service': 'PPTP VPN', 'category': 'vpn', 'risk': 'medium'},
|
||||
3306: {'service': 'MySQL', 'category': 'database', 'risk': 'critical'},
|
||||
3389: {'service': 'RDP', 'category': 'remote_access', 'risk': 'high'},
|
||||
5432: {'service': 'PostgreSQL', 'category': 'database', 'risk': 'critical'},
|
||||
5900: {'service': 'VNC', 'category': 'remote_access', 'risk': 'high'},
|
||||
6379: {'service': 'Redis', 'category': 'database', 'risk': 'critical'},
|
||||
8080: {'service': 'HTTP Proxy', 'category': 'web', 'risk': 'medium'},
|
||||
8443: {'service': 'HTTPS Alt', 'category': 'web', 'risk': 'low'},
|
||||
27017: {'service': 'MongoDB', 'category': 'database', 'risk': 'critical'},
|
||||
}
|
||||
|
||||
# Dangerous ports that should never be exposed
|
||||
self.dangerous_ports = [23, 135, 139, 445, 1433, 1521, 3306, 3389, 5432, 5900, 6379, 27017]
|
||||
|
||||
def scan(self, ip_address: str, ports: List[int] = None) -> Dict:
|
||||
"""
|
||||
Scan ports on IP address
|
||||
|
||||
Args:
|
||||
ip_address: IP address to scan
|
||||
ports: List of ports to scan (default: common ports)
|
||||
|
||||
Returns:
|
||||
Dictionary with scan results
|
||||
"""
|
||||
if ports is None:
|
||||
ports = list(self.common_ports.keys())
|
||||
|
||||
result = {
|
||||
'ip': ip_address,
|
||||
'total_scanned': len(ports),
|
||||
'open_ports': [],
|
||||
'closed_ports': [],
|
||||
'filtered_ports': [],
|
||||
'dangerous_open': [],
|
||||
'services_detected': [],
|
||||
'categories': {},
|
||||
'security_issues': [],
|
||||
'scan_summary': {}
|
||||
}
|
||||
|
||||
# Scan ports in parallel
|
||||
with ThreadPoolExecutor(max_workers=30) as executor:
|
||||
futures = {
|
||||
executor.submit(self._scan_port, ip_address, port): port
|
||||
for port in ports
|
||||
}
|
||||
|
||||
for future in as_completed(futures, timeout=60):
|
||||
port = futures[future]
|
||||
try:
|
||||
status, banner = future.result()
|
||||
|
||||
port_info = self.common_ports.get(port, {
|
||||
'service': 'Unknown',
|
||||
'category': 'other',
|
||||
'risk': 'unknown'
|
||||
})
|
||||
|
||||
port_data = {
|
||||
'port': port,
|
||||
'service': port_info['service'],
|
||||
'category': port_info['category'],
|
||||
'risk_level': port_info['risk'],
|
||||
'banner': banner
|
||||
}
|
||||
|
||||
if status == 'open':
|
||||
result['open_ports'].append(port_data)
|
||||
result['services_detected'].append(port_info['service'])
|
||||
|
||||
# Track by category
|
||||
category = port_info['category']
|
||||
if category not in result['categories']:
|
||||
result['categories'][category] = []
|
||||
result['categories'][category].append(port)
|
||||
|
||||
# Check if dangerous port
|
||||
if port in self.dangerous_ports:
|
||||
result['dangerous_open'].append(port_data)
|
||||
result['security_issues'].append({
|
||||
'severity': 'CRITICAL' if port_info['risk'] == 'critical' else 'HIGH',
|
||||
'port': port,
|
||||
'service': port_info['service'],
|
||||
'issue': f"Dangerous service {port_info['service']} exposed on port {port}"
|
||||
})
|
||||
|
||||
elif status == 'closed':
|
||||
result['closed_ports'].append(port)
|
||||
else:
|
||||
result['filtered_ports'].append(port)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Error scanning port {port}: {e}")
|
||||
result['filtered_ports'].append(port)
|
||||
|
||||
# Generate summary
|
||||
result['scan_summary'] = {
|
||||
'open_count': len(result['open_ports']),
|
||||
'closed_count': len(result['closed_ports']),
|
||||
'filtered_count': len(result['filtered_ports']),
|
||||
'dangerous_count': len(result['dangerous_open']),
|
||||
'has_web': any(p['port'] in [80, 443, 8080, 8443] for p in result['open_ports']),
|
||||
'has_email': any(p['port'] in [25, 110, 143, 465, 587, 993, 995] for p in result['open_ports']),
|
||||
'has_database': any(p['port'] in [3306, 5432, 1433, 1521, 27017, 6379] for p in result['open_ports']),
|
||||
'has_remote_access': any(p['port'] in [22, 23, 3389, 5900] for p in result['open_ports'])
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
def _scan_port(self, ip_address: str, port: int) -> tuple:
|
||||
"""
|
||||
Scan a single port
|
||||
|
||||
Returns:
|
||||
Tuple of (status, banner)
|
||||
status: 'open', 'closed', or 'filtered'
|
||||
"""
|
||||
try:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.settimeout(self.timeout)
|
||||
|
||||
result = sock.connect_ex((ip_address, port))
|
||||
|
||||
if result == 0:
|
||||
# Port is open, try to grab banner
|
||||
banner = self._grab_banner(sock, port)
|
||||
sock.close()
|
||||
return 'open', banner
|
||||
else:
|
||||
sock.close()
|
||||
return 'closed', None
|
||||
|
||||
except socket.timeout:
|
||||
return 'filtered', None
|
||||
except socket.error:
|
||||
return 'filtered', None
|
||||
except Exception as e:
|
||||
return 'filtered', None
|
||||
|
||||
def _grab_banner(self, sock: socket.socket, port: int) -> Optional[str]:
|
||||
"""Try to grab service banner"""
|
||||
try:
|
||||
# For HTTP ports, send a HEAD request
|
||||
if port in [80, 8080]:
|
||||
sock.send(b'HEAD / HTTP/1.0\r\n\r\n')
|
||||
elif port in [443, 8443]:
|
||||
return None # Can't grab banner from SSL without proper handshake
|
||||
else:
|
||||
# For other ports, just try to receive
|
||||
pass
|
||||
|
||||
sock.settimeout(2)
|
||||
banner = sock.recv(1024).decode('utf-8', errors='ignore').strip()
|
||||
return banner[:200] if banner else None # Limit banner length
|
||||
except:
|
||||
return None
|
||||
|
||||
def quick_scan(self, ip_address: str) -> Dict:
|
||||
"""
|
||||
Quick scan of most common web ports
|
||||
|
||||
Args:
|
||||
ip_address: IP to scan
|
||||
|
||||
Returns:
|
||||
Quick scan results
|
||||
"""
|
||||
quick_ports = [21, 22, 23, 25, 80, 443, 3306, 3389, 8080]
|
||||
return self.scan(ip_address, quick_ports)
|
||||
|
||||
def get_port_score(self, scan_data: Dict) -> Dict:
|
||||
"""
|
||||
Calculate security score based on port scan results
|
||||
|
||||
Returns:
|
||||
Dict with score (0-100, higher = more risk) and reasons
|
||||
"""
|
||||
score = 0
|
||||
reasons = []
|
||||
|
||||
# Dangerous ports are critical
|
||||
dangerous_count = len(scan_data.get('dangerous_open', []))
|
||||
if dangerous_count > 0:
|
||||
score += dangerous_count * 20
|
||||
reasons.append(f"{dangerous_count} dangerous port(s) exposed")
|
||||
|
||||
for port in scan_data.get('dangerous_open', []):
|
||||
reasons.append(f" - {port['service']} on port {port['port']}")
|
||||
|
||||
# Database exposed
|
||||
summary = scan_data.get('scan_summary', {})
|
||||
if summary.get('has_database'):
|
||||
score += 30
|
||||
reasons.append("Database port(s) publicly accessible")
|
||||
|
||||
# Remote access (besides SSH)
|
||||
if summary.get('has_remote_access'):
|
||||
open_ports = [p['port'] for p in scan_data.get('open_ports', [])]
|
||||
if 23 in open_ports: # Telnet
|
||||
score += 25
|
||||
reasons.append("Telnet (unencrypted) is open")
|
||||
if 3389 in open_ports: # RDP
|
||||
score += 20
|
||||
reasons.append("RDP is publicly accessible")
|
||||
if 5900 in open_ports: # VNC
|
||||
score += 20
|
||||
reasons.append("VNC is publicly accessible")
|
||||
|
||||
# Too many open ports (attack surface)
|
||||
open_count = summary.get('open_count', 0)
|
||||
if open_count > 10:
|
||||
score += 15
|
||||
reasons.append(f"Large attack surface: {open_count} open ports")
|
||||
elif open_count > 5:
|
||||
score += 5
|
||||
reasons.append(f"{open_count} open ports detected")
|
||||
|
||||
if score == 0:
|
||||
reasons.append("No dangerous services exposed")
|
||||
|
||||
return {
|
||||
'score': min(100, score),
|
||||
'reasons': reasons,
|
||||
'is_secure': score <= 10,
|
||||
'has_critical_issues': dangerous_count > 0 or summary.get('has_database')
|
||||
}
|
||||
741
ai_platform/modules/domain_check/api/app/services/risk_scorer.py
Normal file
741
ai_platform/modules/domain_check/api/app/services/risk_scorer.py
Normal file
|
|
@ -0,0 +1,741 @@
|
|||
"""
|
||||
Risk Scoring Engine - Comprehensive Domain Risk Assessment
|
||||
|
||||
SCORING FORMULA:
|
||||
================
|
||||
Total Risk Score = Σ(Category Score × Category Weight)
|
||||
|
||||
CATEGORIES & WEIGHTS:
|
||||
- Domain Age: 20% (age_score × 0.20)
|
||||
- SSL/TLS: 15% (ssl_score × 0.15)
|
||||
- DNS Config: 10% (dns_score × 0.10)
|
||||
- Email Security: 10% (email_score × 0.10)
|
||||
- WHOIS Privacy: 5% (whois_score × 0.05)
|
||||
- IP Reputation: 10% (ip_score × 0.10)
|
||||
- HTTP Security: 10% (http_score × 0.10)
|
||||
- Blacklists: 15% (blacklist_score × 0.15)
|
||||
- Port Security: 5% (port_score × 0.05)
|
||||
----
|
||||
100%
|
||||
|
||||
INDIVIDUAL SCORE CALCULATIONS (0-100 scale, 0=safe, 100=dangerous):
|
||||
|
||||
1. DOMAIN AGE SCORE:
|
||||
- < 30 days: 100 (CRITICAL)
|
||||
- < 90 days: 80 (HIGH)
|
||||
- < 180 days: 60 (MEDIUM-HIGH)
|
||||
- < 365 days: 40 (MEDIUM)
|
||||
- < 730 days: 20 (LOW)
|
||||
- >= 730 days: 0 (TRUSTED)
|
||||
|
||||
2. SSL/TLS SCORE:
|
||||
- No SSL: 100
|
||||
- Expired: 100
|
||||
- Self-signed: 80
|
||||
- Expires < 7 days: 60
|
||||
- Expires < 30 days: 30
|
||||
- Weak cipher: 40
|
||||
- Valid SSL: 0
|
||||
|
||||
3. DNS SCORE:
|
||||
- No A records: 50
|
||||
- No MX records: 20
|
||||
- No NS at parent: 30
|
||||
- Mismatched NS: 20
|
||||
- No DNSSEC: 10
|
||||
- Complete config: 0
|
||||
|
||||
4. EMAIL SECURITY SCORE:
|
||||
- No SPF: 40
|
||||
- SPF ~all (soft): 20
|
||||
- SPF ?all (neutral): 30
|
||||
- No DKIM: 30
|
||||
- No DMARC: 20
|
||||
- DMARC p=none: 15
|
||||
- Full protection: 0
|
||||
|
||||
5. WHOIS SCORE:
|
||||
- Privacy protected: 20
|
||||
- No registrant info: 15
|
||||
- Suspicious registrar: 30
|
||||
- Free/disposable reg: 40
|
||||
- Transparent: 0
|
||||
|
||||
6. IP REPUTATION SCORE:
|
||||
- High-risk country: 40
|
||||
- Known bad ASN: 50
|
||||
- No reverse DNS: 20
|
||||
- Datacenter IP: 10
|
||||
- Residential IP: 0
|
||||
|
||||
7. HTTP SECURITY SCORE:
|
||||
- No HTTPS: 40
|
||||
- No HSTS: 20
|
||||
- No CSP: 15
|
||||
- No X-Frame-Options: 10
|
||||
- Missing headers: 5 each
|
||||
- Secure config: 0
|
||||
|
||||
8. BLACKLIST SCORE:
|
||||
- On spam blacklist: 25 each
|
||||
- On malware list: 50 each
|
||||
- On phishing list: 60 each
|
||||
- Not listed: 0
|
||||
|
||||
9. PORT SECURITY SCORE:
|
||||
- Database exposed: 30
|
||||
- RDP/VNC exposed: 25
|
||||
- Telnet open: 40
|
||||
- Too many ports (>10): 15
|
||||
- Secure config: 0
|
||||
|
||||
RISK LEVELS:
|
||||
- LOW: 0-25
|
||||
- MEDIUM: 26-50
|
||||
- HIGH: 51-75
|
||||
- CRITICAL: 76-100
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from flask import current_app
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _coerce_datetime(value) -> Optional[datetime]:
|
||||
"""Return a naive datetime for any date-like value, else None.
|
||||
|
||||
Defense-in-depth: even though WhoisService now normalizes dates, the risk
|
||||
scorer can be fed cached/serialized data where dates are ISO strings. Never
|
||||
let ``datetime - <non-datetime>`` raise from here.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
return value.replace(tzinfo=None) if value.tzinfo else value
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
from dateutil import parser as _p
|
||||
dt = _p.parse(text, fuzzy=True)
|
||||
return dt.replace(tzinfo=None) if dt.tzinfo else dt
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
class RiskScorer:
|
||||
"""Calculate comprehensive risk scores for domains"""
|
||||
|
||||
def __init__(self):
|
||||
# Category weights (must sum to 1.0)
|
||||
self.weights = {
|
||||
'domain_age': 0.20,
|
||||
'ssl': 0.15,
|
||||
'dns': 0.10,
|
||||
'email_security': 0.10,
|
||||
'whois': 0.05,
|
||||
'ip_reputation': 0.10,
|
||||
'http_security': 0.10,
|
||||
'blacklist': 0.15,
|
||||
'port_security': 0.05
|
||||
}
|
||||
|
||||
# Risk level thresholds
|
||||
self.thresholds = {
|
||||
'low': 25,
|
||||
'medium': 50,
|
||||
'high': 75
|
||||
}
|
||||
|
||||
def calculate_risk(self, domain_data: Dict) -> Dict:
|
||||
"""
|
||||
Calculate comprehensive risk score
|
||||
|
||||
Args:
|
||||
domain_data: Dictionary containing all domain information
|
||||
- whois: WHOIS data
|
||||
- dns: DNS records
|
||||
- ssl: SSL certificate data
|
||||
- ip_intelligence: IP information
|
||||
- http_analysis: HTTP analysis data
|
||||
- blacklist: Blacklist check results
|
||||
- port_scan: Port scan results
|
||||
|
||||
Returns:
|
||||
Dictionary with complete risk assessment
|
||||
"""
|
||||
factors = []
|
||||
scores = {}
|
||||
formula_breakdown = []
|
||||
|
||||
# 1. Domain Age Score (20%)
|
||||
if domain_data.get('whois'):
|
||||
score, factor = self._score_domain_age(domain_data['whois'])
|
||||
scores['domain_age'] = score
|
||||
factors.append(factor)
|
||||
formula_breakdown.append({
|
||||
'category': 'Domain Age',
|
||||
'raw_score': score,
|
||||
'weight': self.weights['domain_age'],
|
||||
'weighted_score': score * self.weights['domain_age'],
|
||||
'formula': f"{score} × {self.weights['domain_age']} = {score * self.weights['domain_age']:.1f}"
|
||||
})
|
||||
|
||||
# 2. SSL Score (15%)
|
||||
if domain_data.get('ssl'):
|
||||
score, factor = self._score_ssl(domain_data['ssl'])
|
||||
scores['ssl'] = score
|
||||
factors.append(factor)
|
||||
formula_breakdown.append({
|
||||
'category': 'SSL/TLS',
|
||||
'raw_score': score,
|
||||
'weight': self.weights['ssl'],
|
||||
'weighted_score': score * self.weights['ssl'],
|
||||
'formula': f"{score} × {self.weights['ssl']} = {score * self.weights['ssl']:.1f}"
|
||||
})
|
||||
|
||||
# 3. DNS Score (10%)
|
||||
if domain_data.get('dns'):
|
||||
score, factor = self._score_dns(domain_data['dns'])
|
||||
scores['dns'] = score
|
||||
factors.append(factor)
|
||||
formula_breakdown.append({
|
||||
'category': 'DNS Configuration',
|
||||
'raw_score': score,
|
||||
'weight': self.weights['dns'],
|
||||
'weighted_score': score * self.weights['dns'],
|
||||
'formula': f"{score} × {self.weights['dns']} = {score * self.weights['dns']:.1f}"
|
||||
})
|
||||
|
||||
# 4. Email Security Score (10%)
|
||||
if domain_data.get('dns'):
|
||||
score, factor = self._score_email_security(domain_data['dns'])
|
||||
scores['email_security'] = score
|
||||
factors.append(factor)
|
||||
formula_breakdown.append({
|
||||
'category': 'Email Security',
|
||||
'raw_score': score,
|
||||
'weight': self.weights['email_security'],
|
||||
'weighted_score': score * self.weights['email_security'],
|
||||
'formula': f"{score} × {self.weights['email_security']} = {score * self.weights['email_security']:.1f}"
|
||||
})
|
||||
|
||||
# 5. WHOIS Score (5%)
|
||||
if domain_data.get('whois'):
|
||||
score, factor = self._score_whois(domain_data['whois'])
|
||||
scores['whois'] = score
|
||||
factors.append(factor)
|
||||
formula_breakdown.append({
|
||||
'category': 'WHOIS Privacy',
|
||||
'raw_score': score,
|
||||
'weight': self.weights['whois'],
|
||||
'weighted_score': score * self.weights['whois'],
|
||||
'formula': f"{score} × {self.weights['whois']} = {score * self.weights['whois']:.1f}"
|
||||
})
|
||||
|
||||
# 6. IP Reputation Score (10%)
|
||||
if domain_data.get('ip_intelligence'):
|
||||
score, factor = self._score_ip_reputation(domain_data['ip_intelligence'])
|
||||
scores['ip_reputation'] = score
|
||||
factors.append(factor)
|
||||
formula_breakdown.append({
|
||||
'category': 'IP Reputation',
|
||||
'raw_score': score,
|
||||
'weight': self.weights['ip_reputation'],
|
||||
'weighted_score': score * self.weights['ip_reputation'],
|
||||
'formula': f"{score} × {self.weights['ip_reputation']} = {score * self.weights['ip_reputation']:.1f}"
|
||||
})
|
||||
|
||||
# 7. HTTP Security Score (10%)
|
||||
if domain_data.get('http_analysis'):
|
||||
score, factor = self._score_http_security(domain_data['http_analysis'])
|
||||
scores['http_security'] = score
|
||||
factors.append(factor)
|
||||
formula_breakdown.append({
|
||||
'category': 'HTTP Security',
|
||||
'raw_score': score,
|
||||
'weight': self.weights['http_security'],
|
||||
'weighted_score': score * self.weights['http_security'],
|
||||
'formula': f"{score} × {self.weights['http_security']} = {score * self.weights['http_security']:.1f}"
|
||||
})
|
||||
|
||||
# 8. Blacklist Score (15%)
|
||||
if domain_data.get('blacklist'):
|
||||
score, factor = self._score_blacklist(domain_data['blacklist'])
|
||||
scores['blacklist'] = score
|
||||
factors.append(factor)
|
||||
formula_breakdown.append({
|
||||
'category': 'Blacklist Status',
|
||||
'raw_score': score,
|
||||
'weight': self.weights['blacklist'],
|
||||
'weighted_score': score * self.weights['blacklist'],
|
||||
'formula': f"{score} × {self.weights['blacklist']} = {score * self.weights['blacklist']:.1f}"
|
||||
})
|
||||
|
||||
# 9. Port Security Score (5%)
|
||||
if domain_data.get('port_scan'):
|
||||
score, factor = self._score_port_security(domain_data['port_scan'])
|
||||
scores['port_security'] = score
|
||||
factors.append(factor)
|
||||
formula_breakdown.append({
|
||||
'category': 'Port Security',
|
||||
'raw_score': score,
|
||||
'weight': self.weights['port_security'],
|
||||
'weighted_score': score * self.weights['port_security'],
|
||||
'formula': f"{score} × {self.weights['port_security']} = {score * self.weights['port_security']:.1f}"
|
||||
})
|
||||
|
||||
# Calculate weighted total
|
||||
total_weight_used = sum(self.weights[k] for k in scores.keys())
|
||||
if total_weight_used > 0:
|
||||
# Normalize to account for missing checks
|
||||
raw_total = sum(scores[k] * self.weights[k] for k in scores.keys())
|
||||
total_score = (raw_total / total_weight_used) if total_weight_used < 1.0 else raw_total
|
||||
else:
|
||||
total_score = 50 # Default medium risk if no data
|
||||
|
||||
total_score = min(100, max(0, int(total_score)))
|
||||
|
||||
# Determine risk level
|
||||
risk_level = self._get_risk_level(total_score)
|
||||
|
||||
# Generate final formula string
|
||||
formula_string = self._generate_formula_string(scores, total_score)
|
||||
|
||||
return {
|
||||
'total_score': total_score,
|
||||
'risk_level': risk_level,
|
||||
'individual_scores': scores,
|
||||
'factors': factors,
|
||||
'formula_breakdown': formula_breakdown,
|
||||
'formula_string': formula_string,
|
||||
'weights_used': {k: self.weights[k] for k in scores.keys()},
|
||||
'total_weight_used': total_weight_used,
|
||||
'is_new_domain': scores.get('domain_age', 0) >= 80,
|
||||
'is_suspicious': total_score >= 50,
|
||||
'is_blacklisted': scores.get('blacklist', 0) > 0,
|
||||
'requires_manual_review': total_score >= 60,
|
||||
'risk_thresholds': {
|
||||
'low': f"0-{self.thresholds['low']}",
|
||||
'medium': f"{self.thresholds['low']+1}-{self.thresholds['medium']}",
|
||||
'high': f"{self.thresholds['medium']+1}-{self.thresholds['high']}",
|
||||
'critical': f"{self.thresholds['high']+1}-100"
|
||||
}
|
||||
}
|
||||
|
||||
def _score_domain_age(self, whois_data: Dict) -> Tuple[int, Dict]:
|
||||
"""Score based on domain age"""
|
||||
creation_date = _coerce_datetime(whois_data.get('creation_date'))
|
||||
|
||||
if not creation_date:
|
||||
return 50, {
|
||||
'factor': 'domain_age',
|
||||
'score': 50,
|
||||
'weight': self.weights['domain_age'],
|
||||
'weighted_score': 50 * self.weights['domain_age'],
|
||||
'reason': 'Unable to determine domain age',
|
||||
'details': {'creation_date': None, 'age_days': None}
|
||||
}
|
||||
|
||||
age_days = (datetime.utcnow() - creation_date).days
|
||||
|
||||
# Scoring thresholds
|
||||
if age_days < 30:
|
||||
score = 100
|
||||
reason = f'CRITICAL: Very new domain ({age_days} days, <1 month)'
|
||||
elif age_days < 90:
|
||||
score = 80
|
||||
reason = f'HIGH: New domain ({age_days} days, <3 months)'
|
||||
elif age_days < 180:
|
||||
score = 60
|
||||
reason = f'MEDIUM-HIGH: Recently created ({age_days} days, <6 months)'
|
||||
elif age_days < 365:
|
||||
score = 40
|
||||
reason = f'MEDIUM: Less than 1 year old ({age_days} days)'
|
||||
elif age_days < 730:
|
||||
score = 20
|
||||
reason = f'LOW: Established domain ({age_days} days, 1-2 years)'
|
||||
else:
|
||||
score = 0
|
||||
years = age_days // 365
|
||||
reason = f'TRUSTED: Mature domain ({age_days} days, {years}+ years)'
|
||||
|
||||
return score, {
|
||||
'factor': 'domain_age',
|
||||
'score': score,
|
||||
'weight': self.weights['domain_age'],
|
||||
'weighted_score': score * self.weights['domain_age'],
|
||||
'reason': reason,
|
||||
'details': {
|
||||
'creation_date': str(creation_date) if creation_date else None,
|
||||
'age_days': age_days
|
||||
}
|
||||
}
|
||||
|
||||
def _score_ssl(self, ssl_data: Dict) -> Tuple[int, Dict]:
|
||||
"""Score based on SSL/TLS certificate"""
|
||||
if not ssl_data.get('has_ssl'):
|
||||
return 100, {
|
||||
'factor': 'ssl',
|
||||
'score': 100,
|
||||
'weight': self.weights['ssl'],
|
||||
'weighted_score': 100 * self.weights['ssl'],
|
||||
'reason': 'CRITICAL: No SSL/TLS certificate',
|
||||
'details': {'has_ssl': False}
|
||||
}
|
||||
|
||||
score = 0
|
||||
reasons = []
|
||||
|
||||
if ssl_data.get('is_expired'):
|
||||
score = 100
|
||||
reasons.append('Certificate expired')
|
||||
elif ssl_data.get('is_self_signed'):
|
||||
score = 80
|
||||
reasons.append('Self-signed certificate')
|
||||
else:
|
||||
days_until_expiry = ssl_data.get('days_until_expiry', 365)
|
||||
if days_until_expiry < 7:
|
||||
score = 60
|
||||
reasons.append(f'Expires in {days_until_expiry} days')
|
||||
elif days_until_expiry < 30:
|
||||
score = 30
|
||||
reasons.append(f'Expires in {days_until_expiry} days')
|
||||
|
||||
# Check for weak algorithms
|
||||
sig_algo = (ssl_data.get('signature_algorithm') or '').lower()
|
||||
if 'sha1' in sig_algo or 'md5' in sig_algo:
|
||||
score = max(score, 40)
|
||||
reasons.append('Weak signature algorithm')
|
||||
|
||||
if score == 0:
|
||||
reasons.append('Valid SSL certificate')
|
||||
|
||||
return score, {
|
||||
'factor': 'ssl',
|
||||
'score': score,
|
||||
'weight': self.weights['ssl'],
|
||||
'weighted_score': score * self.weights['ssl'],
|
||||
'reason': '; '.join(reasons) if reasons else 'Valid SSL',
|
||||
'details': {
|
||||
'has_ssl': True,
|
||||
'is_valid': ssl_data.get('is_valid'),
|
||||
'days_until_expiry': ssl_data.get('days_until_expiry'),
|
||||
'issuer': ssl_data.get('issuer')
|
||||
}
|
||||
}
|
||||
|
||||
def _score_dns(self, dns_data: Dict) -> Tuple[int, Dict]:
|
||||
"""Score based on DNS configuration"""
|
||||
score = 0
|
||||
reasons = []
|
||||
|
||||
a_records = dns_data.get('a_records', [])
|
||||
if not a_records:
|
||||
score += 50
|
||||
reasons.append('No A records')
|
||||
|
||||
ns_records = dns_data.get('ns_records', [])
|
||||
if not ns_records:
|
||||
score += 30
|
||||
reasons.append('No NS records')
|
||||
|
||||
mx_records = dns_data.get('mx_records', [])
|
||||
if not mx_records:
|
||||
score += 15
|
||||
reasons.append('No MX records')
|
||||
|
||||
# Check for DNSSEC (if available)
|
||||
if dns_data.get('dnssec') == 'inactive' or not dns_data.get('has_dnssec', True):
|
||||
score += 5
|
||||
reasons.append('DNSSEC not enabled')
|
||||
|
||||
if score == 0:
|
||||
reasons.append('Complete DNS configuration')
|
||||
|
||||
return min(100, score), {
|
||||
'factor': 'dns',
|
||||
'score': min(100, score),
|
||||
'weight': self.weights['dns'],
|
||||
'weighted_score': min(100, score) * self.weights['dns'],
|
||||
'reason': '; '.join(reasons),
|
||||
'details': {
|
||||
'a_record_count': len(a_records),
|
||||
'ns_record_count': len(ns_records),
|
||||
'mx_record_count': len(mx_records)
|
||||
}
|
||||
}
|
||||
|
||||
def _score_email_security(self, dns_data: Dict) -> Tuple[int, Dict]:
|
||||
"""Score based on email security (SPF, DKIM, DMARC)"""
|
||||
score = 0
|
||||
reasons = []
|
||||
|
||||
has_spf = dns_data.get('has_spf', False)
|
||||
has_dkim = dns_data.get('has_dkim', False)
|
||||
has_dmarc = dns_data.get('has_dmarc', False)
|
||||
|
||||
if not has_spf:
|
||||
score += 40
|
||||
reasons.append('No SPF record')
|
||||
else:
|
||||
# Check SPF policy strength
|
||||
spf_record = dns_data.get('spf_record', '')
|
||||
if '~all' in spf_record:
|
||||
score += 15
|
||||
reasons.append('SPF uses soft fail (~all)')
|
||||
elif '?all' in spf_record:
|
||||
score += 25
|
||||
reasons.append('SPF uses neutral (?all)')
|
||||
|
||||
if not has_dkim:
|
||||
score += 30
|
||||
reasons.append('No DKIM record detected')
|
||||
|
||||
if not has_dmarc:
|
||||
score += 20
|
||||
reasons.append('No DMARC record')
|
||||
else:
|
||||
dmarc_record = dns_data.get('dmarc_record', '')
|
||||
if 'p=none' in dmarc_record:
|
||||
score += 15
|
||||
reasons.append('DMARC policy is none (monitoring only)')
|
||||
|
||||
if score == 0:
|
||||
reasons.append('Full email security (SPF+DKIM+DMARC)')
|
||||
|
||||
return min(100, score), {
|
||||
'factor': 'email_security',
|
||||
'score': min(100, score),
|
||||
'weight': self.weights['email_security'],
|
||||
'weighted_score': min(100, score) * self.weights['email_security'],
|
||||
'reason': '; '.join(reasons),
|
||||
'details': {
|
||||
'has_spf': has_spf,
|
||||
'has_dkim': has_dkim,
|
||||
'has_dmarc': has_dmarc
|
||||
}
|
||||
}
|
||||
|
||||
def _score_whois(self, whois_data: Dict) -> Tuple[int, Dict]:
|
||||
"""Score based on WHOIS transparency"""
|
||||
score = 0
|
||||
reasons = []
|
||||
|
||||
if whois_data.get('privacy_protected'):
|
||||
score += 20
|
||||
reasons.append('WHOIS privacy protection enabled')
|
||||
|
||||
if not whois_data.get('registrant_org') and not whois_data.get('registrant'):
|
||||
score += 15
|
||||
reasons.append('No registrant information')
|
||||
|
||||
# Check for suspicious registrars
|
||||
registrar = (whois_data.get('registrar') or '').lower()
|
||||
suspicious_registrars = ['namecheap', 'namesilo', 'dynadot', 'freenom']
|
||||
for sus in suspicious_registrars:
|
||||
if sus in registrar:
|
||||
score += 20
|
||||
reasons.append(f'High-risk registrar pattern')
|
||||
break
|
||||
|
||||
if score == 0:
|
||||
reasons.append('Transparent WHOIS information')
|
||||
|
||||
return min(100, score), {
|
||||
'factor': 'whois',
|
||||
'score': min(100, score),
|
||||
'weight': self.weights['whois'],
|
||||
'weighted_score': min(100, score) * self.weights['whois'],
|
||||
'reason': '; '.join(reasons),
|
||||
'details': {
|
||||
'registrar': whois_data.get('registrar'),
|
||||
'privacy_protected': whois_data.get('privacy_protected', False)
|
||||
}
|
||||
}
|
||||
|
||||
def _score_ip_reputation(self, ip_data: Dict) -> Tuple[int, Dict]:
|
||||
"""Score based on IP intelligence"""
|
||||
score = 0
|
||||
reasons = []
|
||||
|
||||
# High-risk countries
|
||||
country_code = ip_data.get('country_code', '')
|
||||
high_risk_countries = ['RU', 'CN', 'KP', 'IR', 'NG', 'VN', 'PK']
|
||||
if country_code in high_risk_countries:
|
||||
score += 40
|
||||
reasons.append(f'High-risk country: {country_code}')
|
||||
|
||||
# No reverse DNS
|
||||
if not ip_data.get('reverse_dns'):
|
||||
score += 20
|
||||
reasons.append('No reverse DNS (PTR)')
|
||||
|
||||
# Datacenter IP (slightly suspicious for some use cases)
|
||||
if ip_data.get('is_datacenter'):
|
||||
score += 10
|
||||
reasons.append('Hosted on datacenter/cloud')
|
||||
|
||||
# Hosting provider scoring
|
||||
hosting_score = ip_data.get('hosting_score', {})
|
||||
if hosting_score.get('is_suspicious'):
|
||||
score += 20
|
||||
reasons.extend(hosting_score.get('reasons', []))
|
||||
|
||||
if score == 0:
|
||||
reasons.append('Clean IP reputation')
|
||||
|
||||
return min(100, score), {
|
||||
'factor': 'ip_reputation',
|
||||
'score': min(100, score),
|
||||
'weight': self.weights['ip_reputation'],
|
||||
'weighted_score': min(100, score) * self.weights['ip_reputation'],
|
||||
'reason': '; '.join(reasons),
|
||||
'details': {
|
||||
'ip': ip_data.get('ip'),
|
||||
'country': ip_data.get('country'),
|
||||
'asn': ip_data.get('asn'),
|
||||
'isp': ip_data.get('isp')
|
||||
}
|
||||
}
|
||||
|
||||
def _score_http_security(self, http_data: Dict) -> Tuple[int, Dict]:
|
||||
"""Score based on HTTP security headers"""
|
||||
score = 0
|
||||
reasons = []
|
||||
|
||||
if not http_data.get('has_https'):
|
||||
score += 40
|
||||
reasons.append('No HTTPS support')
|
||||
|
||||
if http_data.get('has_https') and not http_data.get('http_to_https_redirect'):
|
||||
score += 15
|
||||
reasons.append('No HTTP to HTTPS redirect')
|
||||
|
||||
# Check security headers
|
||||
missing_headers = http_data.get('missing_security_headers', [])
|
||||
for header in missing_headers:
|
||||
severity = header.get('severity', 'LOW')
|
||||
if severity == 'CRITICAL':
|
||||
score += 15
|
||||
elif severity == 'HIGH':
|
||||
score += 10
|
||||
else:
|
||||
score += 5
|
||||
if len(reasons) < 5: # Limit reasons
|
||||
reasons.append(f"Missing: {header.get('header', 'Unknown')}")
|
||||
|
||||
if score == 0:
|
||||
reasons.append('Good HTTP security configuration')
|
||||
|
||||
return min(100, score), {
|
||||
'factor': 'http_security',
|
||||
'score': min(100, score),
|
||||
'weight': self.weights['http_security'],
|
||||
'weighted_score': min(100, score) * self.weights['http_security'],
|
||||
'reason': '; '.join(reasons[:5]),
|
||||
'details': {
|
||||
'has_https': http_data.get('has_https'),
|
||||
'has_hsts': 'Strict-Transport-Security' in http_data.get('security_headers', {}),
|
||||
'missing_headers_count': len(missing_headers)
|
||||
}
|
||||
}
|
||||
|
||||
def _score_blacklist(self, blacklist_data: Dict) -> Tuple[int, Dict]:
|
||||
"""Score based on blacklist checks"""
|
||||
score = 0
|
||||
reasons = []
|
||||
|
||||
if blacklist_data.get('is_blacklisted'):
|
||||
ip_listings = 0
|
||||
domain_listings = 0
|
||||
|
||||
if blacklist_data.get('ip_check'):
|
||||
ip_listings = blacklist_data['ip_check'].get('blacklist_count', 0)
|
||||
score += ip_listings * 25
|
||||
|
||||
if blacklist_data.get('domain_check'):
|
||||
domain_listings = blacklist_data['domain_check'].get('blacklist_count', 0)
|
||||
score += domain_listings * 25
|
||||
|
||||
if ip_listings > 0:
|
||||
reasons.append(f'IP on {ip_listings} blacklist(s)')
|
||||
if domain_listings > 0:
|
||||
reasons.append(f'Domain on {domain_listings} blacklist(s)')
|
||||
else:
|
||||
reasons.append('Not on any blacklists')
|
||||
|
||||
return min(100, score), {
|
||||
'factor': 'blacklist',
|
||||
'score': min(100, score),
|
||||
'weight': self.weights['blacklist'],
|
||||
'weighted_score': min(100, score) * self.weights['blacklist'],
|
||||
'reason': '; '.join(reasons),
|
||||
'details': {
|
||||
'is_blacklisted': blacklist_data.get('is_blacklisted', False),
|
||||
'total_listings': blacklist_data.get('total_listings', 0)
|
||||
}
|
||||
}
|
||||
|
||||
def _score_port_security(self, port_data: Dict) -> Tuple[int, Dict]:
|
||||
"""Score based on port scan results"""
|
||||
score = 0
|
||||
reasons = []
|
||||
|
||||
dangerous_ports = port_data.get('dangerous_open', [])
|
||||
if dangerous_ports:
|
||||
score += len(dangerous_ports) * 20
|
||||
for port in dangerous_ports[:3]: # Limit to 3
|
||||
reasons.append(f"{port['service']} exposed (port {port['port']})")
|
||||
|
||||
summary = port_data.get('scan_summary', {})
|
||||
if summary.get('has_database'):
|
||||
score += 30
|
||||
if 'Database' not in str(reasons):
|
||||
reasons.append('Database port(s) publicly accessible')
|
||||
|
||||
open_count = summary.get('open_count', 0)
|
||||
if open_count > 10:
|
||||
score += 15
|
||||
reasons.append(f'Large attack surface ({open_count} open ports)')
|
||||
|
||||
if score == 0:
|
||||
reasons.append('No dangerous services exposed')
|
||||
|
||||
return min(100, score), {
|
||||
'factor': 'port_security',
|
||||
'score': min(100, score),
|
||||
'weight': self.weights['port_security'],
|
||||
'weighted_score': min(100, score) * self.weights['port_security'],
|
||||
'reason': '; '.join(reasons[:5]),
|
||||
'details': {
|
||||
'open_port_count': summary.get('open_count', 0),
|
||||
'dangerous_port_count': len(dangerous_ports)
|
||||
}
|
||||
}
|
||||
|
||||
def _get_risk_level(self, total_score: int) -> str:
|
||||
"""Determine risk level from total score"""
|
||||
if total_score <= self.thresholds['low']:
|
||||
return 'LOW'
|
||||
elif total_score <= self.thresholds['medium']:
|
||||
return 'MEDIUM'
|
||||
elif total_score <= self.thresholds['high']:
|
||||
return 'HIGH'
|
||||
else:
|
||||
return 'CRITICAL'
|
||||
|
||||
def _generate_formula_string(self, scores: Dict, total: int) -> str:
|
||||
"""Generate human-readable formula string"""
|
||||
parts = []
|
||||
for category, score in scores.items():
|
||||
weight = self.weights.get(category, 0)
|
||||
weighted = score * weight
|
||||
parts.append(f"({score} × {weight})")
|
||||
|
||||
formula = " + ".join(parts)
|
||||
return f"Total = {formula} = {total}"
|
||||
185
ai_platform/modules/domain_check/api/app/services/ssl_service.py
Normal file
185
ai_platform/modules/domain_check/api/app/services/ssl_service.py
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
"""
|
||||
SSL Service - handles SSL certificate validation
|
||||
"""
|
||||
import logging
|
||||
import socket
|
||||
import ssl
|
||||
from datetime import datetime
|
||||
from typing import Dict, Optional
|
||||
from OpenSSL import SSL, crypto
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SSLService:
|
||||
"""SSL certificate checking service"""
|
||||
|
||||
def __init__(self, timeout: int = 10):
|
||||
self.timeout = timeout
|
||||
|
||||
def check(self, domain: str, port: int = 443) -> Optional[Dict]:
|
||||
"""
|
||||
Check SSL certificate for a domain
|
||||
|
||||
Args:
|
||||
domain: Domain name to check
|
||||
port: Port number (default: 443)
|
||||
|
||||
Returns:
|
||||
Dictionary with SSL certificate data or None
|
||||
"""
|
||||
try:
|
||||
logger.info(f'SSL check for: {domain}:{port}')
|
||||
|
||||
# Create SSL context
|
||||
context = ssl.create_default_context()
|
||||
|
||||
# Connect and get certificate
|
||||
with socket.create_connection((domain, port), timeout=self.timeout) as sock:
|
||||
with context.wrap_socket(sock, server_hostname=domain) as ssock:
|
||||
cert_bin = ssock.getpeercert(binary_form=True)
|
||||
cert_dict = ssock.getpeercert()
|
||||
|
||||
# Parse certificate with pyOpenSSL for more details
|
||||
x509 = crypto.load_certificate(crypto.FILETYPE_ASN1, cert_bin)
|
||||
|
||||
ssl_data = {
|
||||
'domain': domain,
|
||||
'has_ssl': True,
|
||||
'is_valid': True,
|
||||
'is_self_signed': self._is_self_signed(x509),
|
||||
'is_expired': False,
|
||||
'is_wildcard': False,
|
||||
'issuer': self._parse_issuer(x509),
|
||||
'subject': self._parse_subject(x509),
|
||||
'serial_number': str(x509.get_serial_number()),
|
||||
'signature_algorithm': x509.get_signature_algorithm().decode('utf-8'),
|
||||
'version': x509.get_version(),
|
||||
'valid_from': None,
|
||||
'valid_until': None,
|
||||
'days_until_expiry': None,
|
||||
'san': self._get_san(x509),
|
||||
'key_size': x509.get_pubkey().bits()
|
||||
}
|
||||
|
||||
# Parse dates
|
||||
not_before = datetime.strptime(
|
||||
x509.get_notBefore().decode('utf-8'),
|
||||
'%Y%m%d%H%M%SZ'
|
||||
)
|
||||
not_after = datetime.strptime(
|
||||
x509.get_notAfter().decode('utf-8'),
|
||||
'%Y%m%d%H%M%SZ'
|
||||
)
|
||||
|
||||
ssl_data['valid_from'] = not_before
|
||||
ssl_data['valid_until'] = not_after
|
||||
|
||||
# Calculate days until expiry
|
||||
days_until = (not_after - datetime.utcnow()).days
|
||||
ssl_data['days_until_expiry'] = days_until
|
||||
|
||||
# Check if expired
|
||||
if days_until < 0:
|
||||
ssl_data['is_expired'] = True
|
||||
ssl_data['is_valid'] = False
|
||||
|
||||
# Check if wildcard
|
||||
subject_cn = self._get_common_name(x509)
|
||||
if subject_cn and subject_cn.startswith('*.'):
|
||||
ssl_data['is_wildcard'] = True
|
||||
|
||||
return ssl_data
|
||||
|
||||
except ssl.SSLError as e:
|
||||
logger.warning(f'SSL error for {domain}: {str(e)}')
|
||||
return {
|
||||
'domain': domain,
|
||||
'has_ssl': True,
|
||||
'is_valid': False,
|
||||
'error': str(e)
|
||||
}
|
||||
except socket.timeout:
|
||||
logger.warning(f'SSL check timeout for {domain}')
|
||||
return {
|
||||
'domain': domain,
|
||||
'has_ssl': False,
|
||||
'error': 'Connection timeout'
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f'SSL check failed for {domain}: {str(e)}')
|
||||
return {
|
||||
'domain': domain,
|
||||
'has_ssl': False,
|
||||
'error': str(e)
|
||||
}
|
||||
|
||||
def _is_self_signed(self, cert: crypto.X509) -> bool:
|
||||
"""Check if certificate is self-signed"""
|
||||
try:
|
||||
issuer = cert.get_issuer()
|
||||
subject = cert.get_subject()
|
||||
return issuer.CN == subject.CN
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _parse_issuer(self, cert: crypto.X509) -> str:
|
||||
"""Parse certificate issuer"""
|
||||
try:
|
||||
issuer = cert.get_issuer()
|
||||
parts = []
|
||||
if hasattr(issuer, 'CN') and issuer.CN:
|
||||
parts.append(f"CN={issuer.CN}")
|
||||
if hasattr(issuer, 'O') and issuer.O:
|
||||
parts.append(f"O={issuer.O}")
|
||||
if hasattr(issuer, 'C') and issuer.C:
|
||||
parts.append(f"C={issuer.C}")
|
||||
return ', '.join(parts) if parts else 'Unknown'
|
||||
except Exception:
|
||||
return 'Unknown'
|
||||
|
||||
def _parse_subject(self, cert: crypto.X509) -> str:
|
||||
"""Parse certificate subject"""
|
||||
try:
|
||||
subject = cert.get_subject()
|
||||
parts = []
|
||||
if hasattr(subject, 'CN') and subject.CN:
|
||||
parts.append(f"CN={subject.CN}")
|
||||
if hasattr(subject, 'O') and subject.O:
|
||||
parts.append(f"O={subject.O}")
|
||||
if hasattr(subject, 'C') and subject.C:
|
||||
parts.append(f"C={subject.C}")
|
||||
return ', '.join(parts) if parts else 'Unknown'
|
||||
except Exception:
|
||||
return 'Unknown'
|
||||
|
||||
def _get_common_name(self, cert: crypto.X509) -> Optional[str]:
|
||||
"""Get Common Name from certificate"""
|
||||
try:
|
||||
subject = cert.get_subject()
|
||||
return subject.CN if hasattr(subject, 'CN') else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _get_san(self, cert: crypto.X509) -> list:
|
||||
"""Get Subject Alternative Names"""
|
||||
try:
|
||||
san_ext = None
|
||||
for i in range(cert.get_extension_count()):
|
||||
ext = cert.get_extension(i)
|
||||
if ext.get_short_name() == b'subjectAltName':
|
||||
san_ext = ext
|
||||
break
|
||||
|
||||
if san_ext:
|
||||
san_str = str(san_ext)
|
||||
# Parse SAN string (format: "DNS:example.com, DNS:www.example.com")
|
||||
sans = []
|
||||
for part in san_str.split(','):
|
||||
part = part.strip()
|
||||
if part.startswith('DNS:'):
|
||||
sans.append(part[4:])
|
||||
return sans
|
||||
except Exception as e:
|
||||
logger.debug(f'Could not parse SAN: {e}')
|
||||
return []
|
||||
|
|
@ -0,0 +1,442 @@
|
|||
"""
|
||||
Subdomain Enumeration Service - Certificate Transparency, DNS enumeration
|
||||
"""
|
||||
import logging
|
||||
import requests
|
||||
import dns.resolver
|
||||
from typing import Dict, List, Set
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SubdomainService:
|
||||
"""Service for discovering subdomains"""
|
||||
|
||||
def __init__(self):
|
||||
self.timeout = 15
|
||||
self.resolver = dns.resolver.Resolver()
|
||||
self.resolver.nameservers = ['8.8.8.8', '8.8.4.4']
|
||||
self.resolver.timeout = 3
|
||||
self.resolver.lifetime = 5
|
||||
|
||||
# Common subdomain prefixes to check (expanded: infra, SaaS, business apps)
|
||||
self.common_subdomains = [
|
||||
'www', 'www2', 'mail', 'webmail', 'smtp', 'pop', 'pop3', 'imap', 'mx', 'mx1', 'mx2',
|
||||
'email', 'mailserver', 'relay', 'newsletter', 'mailgun', 'mailchimp', 'lists', 'list',
|
||||
'ftp', 'sftp', 'ssh', 'admin', 'administrator', 'panel', 'cpanel', 'whm', 'plesk', 'directadmin',
|
||||
'api', 'api1', 'api2', 'apis', 'rest', 'graphql', 'ws', 'sockets', 'dev', 'develop', 'development',
|
||||
'staging', 'stage', 'stg', 'test', 'testing', 'qa', 'uat', 'preprod', 'beta', 'alpha', 'demo', 'sandbox',
|
||||
'blog', 'shop', 'store', 'magento', 'woocommerce', 'checkout', 'pay', 'payment', 'payments', 'billing', 'invoice', 'invoices',
|
||||
'app', 'apps', 'application', 'mobile', 'm', 'web', 'portal', 'my', 'account', 'accounts', 'auth', 'sso', 'login', 'oauth', 'id', 'idp',
|
||||
'cdn', 'static', 'assets', 'img', 'images', 'media', 'video', 'videos', 'stream', 'live', 'download', 'downloads', 'files', 'file', 'share', 'sharepoint',
|
||||
'ns1', 'ns2', 'ns3', 'ns4', 'dns', 'dns1', 'dns2', 'resolver',
|
||||
'vpn', 'remote', 'gateway', 'gw', 'proxy', 'firewall', 'fw', 'router', 'access',
|
||||
'git', 'gitlab', 'github', 'gitea', 'bitbucket', 'svn', 'jenkins', 'ci', 'cicd', 'build', 'registry', 'docker', 'nexus', 'artifactory',
|
||||
'db', 'database', 'mysql', 'postgres', 'pg', 'mongo', 'redis', 'elastic', 'elasticsearch', 'kibana', 'grafana', 'prometheus', 'metrics', 'monitor', 'monitoring', 'status', 'health', 'uptime',
|
||||
'backup', 'backups', 'bk', 'old', 'new', 'v2', 'v3', 'legacy', 'archive',
|
||||
'intranet', 'extranet', 'internal', 'private', 'corp', 'office', 'work',
|
||||
'cloud', 'nextcloud', 'owncloud', 'drive', 'docs', 'doc', 'documents', 'wiki', 'confluence', 'jira', 'kb', 'knowledgebase',
|
||||
'crm', 'erp', 'hr', 'hrm', 'support', 'help', 'helpdesk', 'ticket', 'tickets', 'desk', 'service', 'services', 'client', 'clients', 'customer', 'customers', 'partner', 'partners', 'projects', 'project', 'pm', 'tasks', 'rpa', 'automation', 'tooling', 'tools', 'tool',
|
||||
'chat', 'talk', 'meet', 'conf', 'conference', 'videoconferinta', 'voip', 'pbx', 'sip', 'call', 'calls',
|
||||
'dashboard', 'analytics', 'stats', 'report', 'reports', 'data', 'bi', 'reporting',
|
||||
'autodiscover', 'autoconfig', 'exchange', 'owa', 'mail2', 'webdisk', 'cpcalendars', 'cpcontacts',
|
||||
'vps', 'server', 'server1', 'server2', 'host', 'node', 'node1', 'cluster', 'k8s', 'kubernetes', 'srv',
|
||||
'secure', 'ssl', 'vault', 'secret', 'kms', 'ldap', 'ad', 'radius'
|
||||
]
|
||||
|
||||
def enumerate(self, domain: str, extra_subdomains: List[str] = None) -> Dict:
|
||||
"""
|
||||
Enumerate subdomains using multiple methods.
|
||||
|
||||
Coverage note: CT logs only reveal names that had their own certificate;
|
||||
a wildcard cert (*.domain) or a custom-named record with no cert is
|
||||
invisible to CT. Brute-force only finds names in the wordlist. For a
|
||||
domain you own, the authoritative way to get 100% is the DNS provider's
|
||||
API (e.g. GoDaddy) or AXFR (usually disabled). Pass `extra_subdomains`
|
||||
to verify your own known names directly.
|
||||
|
||||
Args:
|
||||
domain: Root domain to enumerate
|
||||
extra_subdomains: owner-supplied candidate names (bare label or FQDN)
|
||||
|
||||
Returns:
|
||||
Dictionary with discovered subdomains
|
||||
"""
|
||||
result = {
|
||||
'domain': domain,
|
||||
'subdomains': [],
|
||||
'total_found': 0,
|
||||
'sources': {
|
||||
'certificate_transparency': [],
|
||||
'dns_bruteforce': [],
|
||||
'dns_records': [],
|
||||
'zone_transfer': [],
|
||||
'user_provided': []
|
||||
},
|
||||
'has_wildcard_cert': False,
|
||||
'coverage_note': None,
|
||||
'live_subdomains': [],
|
||||
'error': None
|
||||
}
|
||||
|
||||
discovered: Set[str] = set()
|
||||
|
||||
try:
|
||||
# 1. Certificate Transparency logs (crt.sh)
|
||||
ct_raw = self._get_ct_subdomains(domain)
|
||||
result['has_wildcard_cert'] = any(s.startswith('*.') for s in ct_raw)
|
||||
ct_subdomains = [s for s in ct_raw if not s.startswith('*')]
|
||||
result['sources']['certificate_transparency'] = ct_subdomains
|
||||
discovered.update(ct_subdomains)
|
||||
|
||||
# 2. DNS records enumeration (MX/NS/SOA)
|
||||
dns_subdomains = self._get_dns_subdomains(domain)
|
||||
result['sources']['dns_records'] = dns_subdomains
|
||||
discovered.update(dns_subdomains)
|
||||
|
||||
# 3. AXFR zone transfer (jackpot if the NS allows it — usually not)
|
||||
axfr_subdomains = self._try_axfr(domain)
|
||||
result['sources']['zone_transfer'] = axfr_subdomains
|
||||
discovered.update(axfr_subdomains)
|
||||
|
||||
# 4. Bruteforce common subdomains
|
||||
brute_subdomains = self._bruteforce_subdomains(domain)
|
||||
result['sources']['dns_bruteforce'] = brute_subdomains
|
||||
discovered.update(brute_subdomains)
|
||||
|
||||
# 5. Owner-supplied candidate names — verify which actually resolve
|
||||
if extra_subdomains:
|
||||
user_found = self._check_user_subdomains(domain, extra_subdomains)
|
||||
result['sources']['user_provided'] = user_found
|
||||
discovered.update(user_found)
|
||||
|
||||
# Remove wildcards and invalid entries
|
||||
discovered = {
|
||||
s for s in discovered
|
||||
if s and not s.startswith('*') and domain in s
|
||||
}
|
||||
|
||||
result['subdomains'] = sorted(list(discovered))
|
||||
result['total_found'] = len(discovered)
|
||||
|
||||
if result['has_wildcard_cert']:
|
||||
result['coverage_note'] = (
|
||||
'Domeniul are certificat wildcard (*.{0}), deci subdomeniile '
|
||||
'fara certificat propriu NU apar in CT logs. Lista poate fi '
|
||||
'incompleta — foloseste extra_subdomains sau API-ul DNS al '
|
||||
'registrarului (GoDaddy) pentru acoperire 100%.'.format(domain)
|
||||
)
|
||||
|
||||
# Check which subdomains are live
|
||||
result['live_subdomains'] = self._check_live_subdomains(list(discovered)[:60])
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Subdomain enumeration failed for {domain}: {e}")
|
||||
result['error'] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def _try_axfr(self, domain: str) -> List[str]:
|
||||
"""Attempt a DNS zone transfer (AXFR) against each authoritative NS.
|
||||
|
||||
Almost always refused, but when an NS is misconfigured it dumps the
|
||||
entire zone — every subdomain at once. Cheap to try, big payoff."""
|
||||
import dns.query
|
||||
import dns.zone
|
||||
found: Set[str] = set()
|
||||
try:
|
||||
ns_records = self.resolver.resolve(domain, 'NS')
|
||||
nameservers = [str(ns.target).rstrip('.') for ns in ns_records]
|
||||
except Exception:
|
||||
return []
|
||||
for ns in nameservers[:4]:
|
||||
try:
|
||||
ns_ip = str(self.resolver.resolve(ns, 'A')[0])
|
||||
zone = dns.zone.from_xfr(dns.query.xfr(ns_ip, domain, timeout=5, lifetime=8))
|
||||
for name in zone.nodes.keys():
|
||||
label = str(name)
|
||||
if label in ('@', ''):
|
||||
continue
|
||||
fqdn = f'{label}.{domain}' if not label.endswith(domain) else label
|
||||
found.add(fqdn.rstrip('.').lower())
|
||||
logger.info(f'AXFR succeeded against {ns} for {domain} ({len(found)} names)')
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
return list(found)
|
||||
|
||||
def _check_user_subdomains(self, domain: str, names: List[str]) -> List[str]:
|
||||
"""Resolve owner-supplied candidate names and keep the ones that exist."""
|
||||
candidates = []
|
||||
for n in names:
|
||||
n = str(n).strip().lower().rstrip('.')
|
||||
if not n:
|
||||
continue
|
||||
fqdn = n if n.endswith(domain) else f'{n}.{domain}'
|
||||
candidates.append(fqdn)
|
||||
found = []
|
||||
with ThreadPoolExecutor(max_workers=20) as ex:
|
||||
futures = {ex.submit(self._resolves, c): c for c in candidates}
|
||||
for fut in as_completed(futures, timeout=30):
|
||||
try:
|
||||
if fut.result():
|
||||
found.append(futures[fut])
|
||||
except Exception:
|
||||
pass
|
||||
return found
|
||||
|
||||
def _resolves(self, name: str) -> bool:
|
||||
for rtype in ('A', 'AAAA', 'CNAME'):
|
||||
try:
|
||||
self.resolver.resolve(name, rtype)
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
return False
|
||||
|
||||
def _get_ct_subdomains(self, domain: str) -> List[str]:
|
||||
"""Get subdomains from Certificate Transparency logs.
|
||||
|
||||
crt.sh is the richest source for real (incl. custom-named) subdomains but
|
||||
is frequently slow or rate-limited, so we retry, and fall back to the
|
||||
certspotter API when crt.sh keeps failing."""
|
||||
subdomains: Set[str] = set()
|
||||
|
||||
# Primary: crt.sh, with retries (it flakes often).
|
||||
for attempt in range(3):
|
||||
try:
|
||||
response = requests.get(
|
||||
f"https://crt.sh/?q=%.{domain}&output=json",
|
||||
timeout=self.timeout,
|
||||
headers={'User-Agent': 'domain-check/1.0'}
|
||||
)
|
||||
if response.status_code == 200 and response.text.strip():
|
||||
for entry in response.json():
|
||||
for name in entry.get('name_value', '').split('\n'):
|
||||
name = name.strip().lower()
|
||||
if name and domain in name:
|
||||
subdomains.add(name)
|
||||
if subdomains:
|
||||
return list(subdomains)
|
||||
except Exception as e:
|
||||
logger.warning(f"crt.sh attempt {attempt + 1} failed: {e}")
|
||||
|
||||
# Fallback: Cert Spotter (no key needed for low volume).
|
||||
try:
|
||||
resp = requests.get(
|
||||
f"https://api.certspotter.com/v1/issuances?domain={domain}"
|
||||
"&include_subdomains=true&expand=dns_names",
|
||||
timeout=self.timeout, headers={'User-Agent': 'domain-check/1.0'}
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
for cert in resp.json():
|
||||
for name in cert.get('dns_names', []):
|
||||
name = str(name).strip().lower()
|
||||
if name and domain in name:
|
||||
subdomains.add(name)
|
||||
except Exception as e:
|
||||
logger.warning(f"certspotter fallback failed: {e}")
|
||||
|
||||
return list(subdomains)
|
||||
|
||||
def _get_dns_subdomains(self, domain: str) -> List[str]:
|
||||
"""Get subdomains from DNS records (NS, MX, etc.)"""
|
||||
subdomains = []
|
||||
|
||||
try:
|
||||
# Check MX records
|
||||
try:
|
||||
mx_records = self.resolver.resolve(domain, 'MX')
|
||||
for mx in mx_records:
|
||||
host = str(mx.exchange).rstrip('.')
|
||||
if domain in host:
|
||||
subdomains.append(host)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Check NS records
|
||||
try:
|
||||
ns_records = self.resolver.resolve(domain, 'NS')
|
||||
for ns in ns_records:
|
||||
host = str(ns.target).rstrip('.')
|
||||
if domain in host:
|
||||
subdomains.append(host)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Check SOA record
|
||||
try:
|
||||
soa_records = self.resolver.resolve(domain, 'SOA')
|
||||
for soa in soa_records:
|
||||
mname = str(soa.mname).rstrip('.')
|
||||
if domain in mname:
|
||||
subdomains.append(mname)
|
||||
except:
|
||||
pass
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"DNS subdomain lookup failed: {e}")
|
||||
|
||||
return list(set(subdomains))
|
||||
|
||||
def _bruteforce_subdomains(self, domain: str) -> List[str]:
|
||||
"""Bruteforce common subdomains"""
|
||||
found = []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=20) as executor:
|
||||
futures = {
|
||||
executor.submit(self._check_subdomain, f"{sub}.{domain}"): sub
|
||||
for sub in self.common_subdomains
|
||||
}
|
||||
|
||||
for future in as_completed(futures, timeout=30):
|
||||
subdomain = futures[future]
|
||||
try:
|
||||
full_subdomain = f"{subdomain}.{domain}"
|
||||
if future.result():
|
||||
found.append(full_subdomain)
|
||||
except:
|
||||
pass
|
||||
|
||||
return found
|
||||
|
||||
def _check_subdomain(self, subdomain: str) -> bool:
|
||||
"""Check if subdomain resolves"""
|
||||
try:
|
||||
self.resolver.resolve(subdomain, 'A')
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
|
||||
def _check_live_subdomains(self, subdomains: List[str]) -> List[Dict]:
|
||||
"""Check which subdomains are live (have HTTP response)"""
|
||||
live = []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||
futures = {
|
||||
executor.submit(self._check_http, sub): sub
|
||||
for sub in subdomains
|
||||
}
|
||||
|
||||
for future in as_completed(futures, timeout=60):
|
||||
subdomain = futures[future]
|
||||
try:
|
||||
result = future.result()
|
||||
if result:
|
||||
live.append(result)
|
||||
except:
|
||||
pass
|
||||
|
||||
return live
|
||||
|
||||
def _check_http(self, subdomain: str) -> Dict:
|
||||
"""Check HTTP/HTTPS on subdomain"""
|
||||
result = {
|
||||
'subdomain': subdomain,
|
||||
'ip': None,
|
||||
'http': None,
|
||||
'https': None
|
||||
}
|
||||
|
||||
# Get IP
|
||||
try:
|
||||
answers = self.resolver.resolve(subdomain, 'A')
|
||||
result['ip'] = str(answers[0])
|
||||
except:
|
||||
return None
|
||||
|
||||
# Check HTTPS
|
||||
try:
|
||||
response = requests.head(
|
||||
f"https://{subdomain}",
|
||||
timeout=5,
|
||||
allow_redirects=True,
|
||||
verify=False
|
||||
)
|
||||
result['https'] = response.status_code
|
||||
except:
|
||||
pass
|
||||
|
||||
# Check HTTP
|
||||
try:
|
||||
response = requests.head(
|
||||
f"http://{subdomain}",
|
||||
timeout=5,
|
||||
allow_redirects=False
|
||||
)
|
||||
result['http'] = response.status_code
|
||||
except:
|
||||
pass
|
||||
|
||||
if result['http'] or result['https']:
|
||||
return result
|
||||
return None
|
||||
|
||||
def check_subdomain_takeover(self, subdomains: List[str]) -> List[Dict]:
|
||||
"""
|
||||
Check for potential subdomain takeover vulnerabilities
|
||||
|
||||
Args:
|
||||
subdomains: List of subdomains to check
|
||||
|
||||
Returns:
|
||||
List of potentially vulnerable subdomains
|
||||
"""
|
||||
vulnerable = []
|
||||
|
||||
# CNAME fingerprints for takeover
|
||||
takeover_fingerprints = {
|
||||
'github.io': 'GitHub Pages',
|
||||
'herokuapp.com': 'Heroku',
|
||||
'herokudns.com': 'Heroku',
|
||||
'wordpress.com': 'WordPress',
|
||||
'pantheonsite.io': 'Pantheon',
|
||||
'domains.tumblr.com': 'Tumblr',
|
||||
'zendesk.com': 'Zendesk',
|
||||
'shopify.com': 'Shopify',
|
||||
'myshopify.com': 'Shopify',
|
||||
's3.amazonaws.com': 'AWS S3',
|
||||
's3-website': 'AWS S3',
|
||||
'cloudfront.net': 'AWS CloudFront',
|
||||
'azurewebsites.net': 'Azure',
|
||||
'cloudapp.net': 'Azure',
|
||||
'trafficmanager.net': 'Azure',
|
||||
'blob.core.windows.net': 'Azure Blob',
|
||||
'ghost.io': 'Ghost',
|
||||
'helpjuice.com': 'Helpjuice',
|
||||
'helpscoutdocs.com': 'HelpScout',
|
||||
'freshdesk.com': 'Freshdesk',
|
||||
'surge.sh': 'Surge',
|
||||
'bitbucket.io': 'Bitbucket',
|
||||
'uservoice.com': 'UserVoice',
|
||||
'simplebooklet.com': 'Simplebooklet'
|
||||
}
|
||||
|
||||
for subdomain in subdomains:
|
||||
try:
|
||||
# Check for CNAME
|
||||
cname_records = self.resolver.resolve(subdomain, 'CNAME')
|
||||
for cname in cname_records:
|
||||
cname_target = str(cname.target).rstrip('.').lower()
|
||||
|
||||
for fingerprint, service in takeover_fingerprints.items():
|
||||
if fingerprint in cname_target:
|
||||
# Check if CNAME target resolves
|
||||
try:
|
||||
self.resolver.resolve(cname_target, 'A')
|
||||
except dns.resolver.NXDOMAIN:
|
||||
vulnerable.append({
|
||||
'subdomain': subdomain,
|
||||
'cname': cname_target,
|
||||
'service': service,
|
||||
'status': 'VULNERABLE',
|
||||
'reason': 'CNAME points to non-existent resource'
|
||||
})
|
||||
except:
|
||||
pass
|
||||
break
|
||||
except:
|
||||
pass
|
||||
|
||||
return vulnerable
|
||||
|
|
@ -0,0 +1,248 @@
|
|||
"""
|
||||
WHOIS Service - handles WHOIS/RDAP lookups
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Union
|
||||
|
||||
import requests
|
||||
|
||||
try:
|
||||
from dateutil import parser as dateutil_parser
|
||||
DATEUTIL_AVAILABLE = True
|
||||
except ImportError:
|
||||
DATEUTIL_AVAILABLE = False
|
||||
|
||||
try:
|
||||
import whois as python_whois
|
||||
WHOIS_AVAILABLE = True
|
||||
except ImportError:
|
||||
WHOIS_AVAILABLE = False
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def normalize_whois_date(value) -> Optional[datetime]:
|
||||
"""Coerce any WHOIS date representation into a naive (tz-stripped) datetime.
|
||||
|
||||
python-whois and registry-specific WHOIS servers (notably ROTLD for .ro)
|
||||
return dates as datetime, list-of-datetime, ISO strings, or arbitrary
|
||||
free-form strings. Downstream code does ``datetime - value`` arithmetic, so
|
||||
every date MUST arrive as a ``datetime`` or ``None`` — never a raw string.
|
||||
A raw string here is the root cause of the historical 500 on ``.ro`` domains.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, list):
|
||||
# Registries often return the most recent date first; take the first set value.
|
||||
for item in value:
|
||||
parsed = normalize_whois_date(item)
|
||||
if parsed is not None:
|
||||
return parsed
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
return value.replace(tzinfo=None) if value.tzinfo else value
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
if not text or text.lower() in ('none', 'null', 'n/a', '-'):
|
||||
return None
|
||||
if DATEUTIL_AVAILABLE:
|
||||
try:
|
||||
parsed = dateutil_parser.parse(text, fuzzy=True)
|
||||
return parsed.replace(tzinfo=None) if parsed.tzinfo else parsed
|
||||
except (ValueError, OverflowError, TypeError):
|
||||
return None
|
||||
# Fallback without dateutil: a few common explicit formats.
|
||||
for fmt in ('%Y-%m-%dT%H:%M:%S.%fZ', '%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%dT%H:%M:%S',
|
||||
'%Y-%m-%d %H:%M:%S', '%Y-%m-%d', '%d.%m.%Y', '%d-%b-%Y'):
|
||||
try:
|
||||
return datetime.strptime(text, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _as_list(value) -> List:
|
||||
"""Normalize a WHOIS field that may be a scalar, None, or list into a clean list."""
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
return [v for v in value if v not in (None, '')]
|
||||
return [value]
|
||||
|
||||
|
||||
class WhoisService:
|
||||
"""WHOIS/RDAP service class"""
|
||||
|
||||
def __init__(self, whoxy_api_key: str = None):
|
||||
self.whoxy_api_key = whoxy_api_key
|
||||
self.whoxy_url = 'https://api.whoxy.com/'
|
||||
|
||||
def lookup(self, domain: str, source: str = 'auto') -> Optional[Dict]:
|
||||
"""
|
||||
Perform WHOIS lookup
|
||||
|
||||
Args:
|
||||
domain: Domain name to lookup
|
||||
source: 'rdap', 'whoxy', or 'auto' (tries RDAP first, falls back to Whoxy)
|
||||
|
||||
Returns:
|
||||
Dictionary with WHOIS data or None
|
||||
"""
|
||||
if source == 'auto':
|
||||
# Try RDAP first
|
||||
result = self._lookup_rdap(domain)
|
||||
# Check if we got useful data (creation_date should exist)
|
||||
if result and result.get('creation_date'):
|
||||
logger.info(f'Using WHOIS data from python-whois for {domain}')
|
||||
return result
|
||||
|
||||
# Fallback to Whoxy if available
|
||||
if self.whoxy_api_key:
|
||||
logger.info(f'Falling back to Whoxy API for {domain}')
|
||||
whoxy_result = self._lookup_whoxy(domain)
|
||||
if whoxy_result:
|
||||
return whoxy_result
|
||||
|
||||
# If we got partial data from RDAP, return it
|
||||
if result:
|
||||
logger.warning(f'Returning incomplete WHOIS data for {domain}')
|
||||
return result
|
||||
|
||||
return None
|
||||
|
||||
elif source == 'rdap':
|
||||
return self._lookup_rdap(domain)
|
||||
|
||||
elif source == 'whoxy':
|
||||
return self._lookup_whoxy(domain)
|
||||
|
||||
return None
|
||||
|
||||
def _lookup_rdap(self, domain: str) -> Optional[Dict]:
|
||||
"""Lookup using python-whois library"""
|
||||
if not WHOIS_AVAILABLE:
|
||||
logger.warning('python-whois not available')
|
||||
return None
|
||||
|
||||
try:
|
||||
logger.info(f'WHOIS lookup for: {domain}')
|
||||
w = python_whois.whois(domain)
|
||||
|
||||
# Normalize every date to datetime|None so downstream arithmetic is safe.
|
||||
creation_date = normalize_whois_date(getattr(w, 'creation_date', None))
|
||||
expiration_date = normalize_whois_date(getattr(w, 'expiration_date', None))
|
||||
updated_date = normalize_whois_date(getattr(w, 'updated_date', None))
|
||||
|
||||
name_servers = [str(ns).lower() for ns in _as_list(getattr(w, 'name_servers', None))]
|
||||
status = [str(s) for s in _as_list(getattr(w, 'status', None))]
|
||||
emails = _as_list(getattr(w, 'emails', None))
|
||||
|
||||
result = {
|
||||
'domain': domain,
|
||||
'creation_date': creation_date,
|
||||
'expiration_date': expiration_date,
|
||||
'updated_date': updated_date,
|
||||
'registrar': getattr(w, 'registrar', None),
|
||||
'registrar_url': None,
|
||||
'registrant_org': getattr(w, 'org', None),
|
||||
'registrant_country': getattr(w, 'country', None),
|
||||
'admin_email': emails[0] if emails else None,
|
||||
'name_servers': name_servers,
|
||||
'status': status,
|
||||
'dnssec': None,
|
||||
'data_source': 'whois',
|
||||
'raw_data': {}
|
||||
}
|
||||
result['is_registered'] = self._is_registered(result)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'WHOIS lookup failed for {domain}: {str(e)}')
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _is_registered(whois_result: Dict) -> Optional[bool]:
|
||||
"""Best-effort determination of whether a domain is registered.
|
||||
|
||||
Returns True/False from WHOIS signals, or None when WHOIS is too sparse
|
||||
to decide (common for ROTLD .ro) — the route then refines this using DNS.
|
||||
"""
|
||||
if whois_result.get('creation_date') or whois_result.get('registrar'):
|
||||
return True
|
||||
if whois_result.get('name_servers') or whois_result.get('status'):
|
||||
return True
|
||||
# No positive WHOIS signal at all: likely available, but let DNS confirm.
|
||||
return None
|
||||
|
||||
def _lookup_whoxy(self, domain: str) -> Optional[Dict]:
|
||||
"""Lookup using Whoxy API"""
|
||||
if not self.whoxy_api_key:
|
||||
logger.warning('Whoxy API key not configured')
|
||||
return None
|
||||
|
||||
try:
|
||||
logger.info(f'Whoxy API lookup for: {domain}')
|
||||
|
||||
response = requests.get(
|
||||
self.whoxy_url,
|
||||
params={
|
||||
'key': self.whoxy_api_key,
|
||||
'whois': domain
|
||||
},
|
||||
timeout=30
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.error(f'Whoxy API error: {response.status_code}')
|
||||
return None
|
||||
|
||||
data = response.json()
|
||||
|
||||
if data.get('status') != 1:
|
||||
logger.error(f'Whoxy API returned error: {data.get("status_reason")}')
|
||||
return None
|
||||
|
||||
whois_data = data.get('whois_data', {})
|
||||
|
||||
result = {
|
||||
'domain': domain,
|
||||
'creation_date': normalize_whois_date(whois_data.get('create_date')),
|
||||
'expiration_date': normalize_whois_date(whois_data.get('expiry_date')),
|
||||
'updated_date': normalize_whois_date(whois_data.get('update_date')),
|
||||
'registrar': whois_data.get('registrar_name'),
|
||||
'registrar_url': whois_data.get('registrar_url'),
|
||||
'registrant_org': whois_data.get('registrant_organization'),
|
||||
'registrant_country': whois_data.get('registrant_country'),
|
||||
'admin_email': whois_data.get('admin_email'),
|
||||
'name_servers': [str(ns).lower() for ns in _as_list(whois_data.get('name_servers'))],
|
||||
'status': [str(s) for s in _as_list(whois_data.get('domain_status'))],
|
||||
'dnssec': whois_data.get('dnssec') == 'yes',
|
||||
'data_source': 'whoxy',
|
||||
'raw_data': whois_data
|
||||
}
|
||||
result['is_registered'] = self._is_registered(result)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Whoxy lookup failed for {domain}: {str(e)}')
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _parse_date(date_str: str) -> Optional[datetime]:
|
||||
"""Parse date string to datetime"""
|
||||
if not date_str:
|
||||
return None
|
||||
|
||||
try:
|
||||
# Try common date formats
|
||||
for fmt in ['%Y-%m-%d', '%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%d %H:%M:%S']:
|
||||
try:
|
||||
return datetime.strptime(date_str, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
637
ai_platform/modules/domain_check/api/app/static/index.html
Normal file
637
ai_platform/modules/domain_check/api/app/static/index.html
Normal file
|
|
@ -0,0 +1,637 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="ro">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Domain Check - Verificare Completă Domenii</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body {
|
||||
font-family: 'Segoe UI', system-ui, sans-serif;
|
||||
background: linear-gradient(135deg, #1a1a2e 0%, #16213e 50%, #0f3460 100%);
|
||||
min-height: 100vh;
|
||||
color: #e4e4e4;
|
||||
}
|
||||
.container { max-width: 1400px; margin: 0 auto; padding: 20px; }
|
||||
h1 {
|
||||
text-align: center;
|
||||
font-size: 2rem;
|
||||
margin-bottom: 10px;
|
||||
background: linear-gradient(90deg, #00d9ff, #00ff88);
|
||||
-webkit-background-clip: text;
|
||||
-webkit-text-fill-color: transparent;
|
||||
}
|
||||
.subtitle { text-align: center; color: #888; margin-bottom: 30px; font-size: 0.9rem; }
|
||||
|
||||
.search-box {
|
||||
background: rgba(255,255,255,0.05);
|
||||
border-radius: 16px;
|
||||
padding: 25px;
|
||||
margin-bottom: 25px;
|
||||
border: 1px solid rgba(255,255,255,0.1);
|
||||
}
|
||||
.input-group {
|
||||
display: flex;
|
||||
gap: 15px;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
}
|
||||
.input-group input {
|
||||
flex: 1;
|
||||
min-width: 250px;
|
||||
padding: 15px 20px;
|
||||
border: 2px solid rgba(255,255,255,0.2);
|
||||
border-radius: 10px;
|
||||
font-size: 1.1rem;
|
||||
background: rgba(0,0,0,0.3);
|
||||
color: #fff;
|
||||
}
|
||||
.input-group input:focus { outline: none; border-color: #00d9ff; }
|
||||
.btn {
|
||||
padding: 15px 35px;
|
||||
border: none;
|
||||
border-radius: 10px;
|
||||
font-size: 1rem;
|
||||
cursor: pointer;
|
||||
font-weight: 600;
|
||||
transition: all 0.3s;
|
||||
}
|
||||
.btn-primary {
|
||||
background: linear-gradient(135deg, #00d9ff, #00ff88);
|
||||
color: #1a1a2e;
|
||||
}
|
||||
.btn-primary:hover { transform: translateY(-2px); box-shadow: 0 5px 20px rgba(0,217,255,0.4); }
|
||||
.btn-primary:disabled { opacity: 0.5; cursor: not-allowed; transform: none; }
|
||||
|
||||
.options {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 15px;
|
||||
margin-top: 20px;
|
||||
padding-top: 20px;
|
||||
border-top: 1px solid rgba(255,255,255,0.1);
|
||||
}
|
||||
.option-group { display: flex; align-items: center; gap: 8px; }
|
||||
.option-group input[type="checkbox"] { width: 18px; height: 18px; cursor: pointer; }
|
||||
.option-group label { cursor: pointer; font-size: 0.9rem; }
|
||||
.option-group.slow label { color: #ff9800; }
|
||||
|
||||
.loading { display: none; text-align: center; padding: 50px; }
|
||||
.spinner {
|
||||
width: 50px; height: 50px;
|
||||
border: 4px solid rgba(255,255,255,0.1);
|
||||
border-top-color: #00d9ff;
|
||||
border-radius: 50%;
|
||||
animation: spin 1s linear infinite;
|
||||
margin: 0 auto 20px;
|
||||
}
|
||||
@keyframes spin { to { transform: rotate(360deg); } }
|
||||
|
||||
.results { display: none; }
|
||||
.results-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(400px, 1fr));
|
||||
gap: 20px;
|
||||
}
|
||||
|
||||
.card {
|
||||
background: rgba(255,255,255,0.05);
|
||||
border-radius: 12px;
|
||||
border: 1px solid rgba(255,255,255,0.1);
|
||||
overflow: hidden;
|
||||
}
|
||||
.card-header {
|
||||
padding: 15px 20px;
|
||||
background: rgba(0,0,0,0.3);
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
border-bottom: 1px solid rgba(255,255,255,0.1);
|
||||
}
|
||||
.card-header h3 { font-size: 1rem; display: flex; align-items: center; gap: 10px; }
|
||||
.card-header .source { font-size: 0.75rem; color: #888; background: rgba(255,255,255,0.1); padding: 3px 8px; border-radius: 4px; }
|
||||
.card-body { padding: 20px; }
|
||||
|
||||
.risk-card { grid-column: 1 / -1; }
|
||||
.risk-score-display {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 30px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.risk-circle {
|
||||
width: 150px; height: 150px;
|
||||
border-radius: 50%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: 3rem;
|
||||
font-weight: bold;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.risk-circle .label { font-size: 0.9rem; font-weight: normal; margin-top: 5px; }
|
||||
.risk-low { background: linear-gradient(135deg, #00c853, #00e676); color: #1a1a2e; }
|
||||
.risk-medium { background: linear-gradient(135deg, #ffc107, #ffeb3b); color: #1a1a2e; }
|
||||
.risk-high { background: linear-gradient(135deg, #ff5722, #ff9800); color: #fff; }
|
||||
.risk-critical { background: linear-gradient(135deg, #d32f2f, #f44336); color: #fff; }
|
||||
|
||||
.risk-details { flex: 1; min-width: 300px; }
|
||||
.risk-factor {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 10px 0;
|
||||
border-bottom: 1px solid rgba(255,255,255,0.1);
|
||||
}
|
||||
.risk-factor:last-child { border-bottom: none; }
|
||||
.risk-factor .name { font-weight: 500; }
|
||||
.risk-factor .score {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
}
|
||||
.risk-factor .bar {
|
||||
width: 100px;
|
||||
height: 8px;
|
||||
background: rgba(255,255,255,0.1);
|
||||
border-radius: 4px;
|
||||
overflow: hidden;
|
||||
}
|
||||
.risk-factor .bar-fill { height: 100%; transition: width 0.5s; }
|
||||
.risk-factor .value { width: 40px; text-align: right; font-weight: bold; }
|
||||
|
||||
.formula-box {
|
||||
background: rgba(0,0,0,0.3);
|
||||
border-radius: 8px;
|
||||
padding: 15px;
|
||||
margin-top: 20px;
|
||||
font-family: monospace;
|
||||
font-size: 0.85rem;
|
||||
overflow-x: auto;
|
||||
}
|
||||
.formula-box h4 { margin-bottom: 10px; color: #00d9ff; }
|
||||
.formula-line { padding: 3px 0; }
|
||||
|
||||
.data-row {
|
||||
display: flex;
|
||||
padding: 10px 0;
|
||||
border-bottom: 1px solid rgba(255,255,255,0.05);
|
||||
}
|
||||
.data-row:last-child { border-bottom: none; }
|
||||
.data-label { width: 140px; color: #888; font-size: 0.9rem; flex-shrink: 0; }
|
||||
.data-value { flex: 1; word-break: break-all; }
|
||||
|
||||
.badge {
|
||||
display: inline-block;
|
||||
padding: 3px 10px;
|
||||
border-radius: 4px;
|
||||
font-size: 0.8rem;
|
||||
margin: 2px;
|
||||
}
|
||||
.badge.success { background: rgba(0,200,83,0.2); color: #00c853; }
|
||||
.badge.danger { background: rgba(244,67,54,0.2); color: #f44336; }
|
||||
.badge.warning { background: rgba(255,152,0,0.2); color: #ff9800; }
|
||||
.badge.info { background: rgba(0,217,255,0.2); color: #00d9ff; }
|
||||
.badge.neutral { background: rgba(255,255,255,0.1); color: #aaa; }
|
||||
|
||||
.port-list { display: flex; flex-wrap: wrap; gap: 5px; }
|
||||
.port-item {
|
||||
padding: 5px 10px;
|
||||
border-radius: 4px;
|
||||
font-size: 0.8rem;
|
||||
background: rgba(255,255,255,0.1);
|
||||
}
|
||||
.port-item.danger { background: rgba(244,67,54,0.2); color: #f44336; }
|
||||
.port-item.warning { background: rgba(255,152,0,0.2); color: #ff9800; }
|
||||
.port-item.safe { background: rgba(0,200,83,0.2); color: #00c853; }
|
||||
|
||||
.blacklist-item {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
padding: 8px 0;
|
||||
border-bottom: 1px solid rgba(255,255,255,0.05);
|
||||
}
|
||||
.blacklist-clean { color: #00c853; }
|
||||
.blacklist-listed { color: #f44336; }
|
||||
|
||||
.error {
|
||||
background: rgba(244,67,54,0.2);
|
||||
border: 1px solid #f44336;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
text-align: center;
|
||||
color: #f44336;
|
||||
}
|
||||
|
||||
.tech-tags { display: flex; flex-wrap: wrap; gap: 8px; }
|
||||
.tech-tag {
|
||||
background: linear-gradient(135deg, rgba(0,217,255,0.2), rgba(0,255,136,0.2));
|
||||
padding: 5px 12px;
|
||||
border-radius: 20px;
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
.security-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 8px 0;
|
||||
border-bottom: 1px solid rgba(255,255,255,0.05);
|
||||
}
|
||||
.header-present { color: #00c853; }
|
||||
.header-missing { color: #f44336; }
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.results-grid { grid-template-columns: 1fr; }
|
||||
.risk-score-display { flex-direction: column; text-align: center; }
|
||||
.input-group { flex-direction: column; }
|
||||
.input-group input { width: 100%; }
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>🔍 Domain Check</h1>
|
||||
<p class="subtitle">Verificare completă domenii - WHOIS, DNS, SSL, IP Intelligence, HTTP Security, Blacklists</p>
|
||||
|
||||
<div class="search-box">
|
||||
<div class="input-group">
|
||||
<input type="text" id="domainInput" placeholder="Introdu domeniul (ex: google.com, example.ro)" autofocus>
|
||||
<button class="btn btn-primary" id="checkBtn" onclick="checkDomain()">Verifică Domeniul</button>
|
||||
</div>
|
||||
|
||||
<div class="options">
|
||||
<div class="option-group">
|
||||
<input type="checkbox" id="optWhois" checked>
|
||||
<label for="optWhois">WHOIS</label>
|
||||
</div>
|
||||
<div class="option-group">
|
||||
<input type="checkbox" id="optDns" checked>
|
||||
<label for="optDns">DNS</label>
|
||||
</div>
|
||||
<div class="option-group">
|
||||
<input type="checkbox" id="optSsl" checked>
|
||||
<label for="optSsl">SSL/TLS</label>
|
||||
</div>
|
||||
<div class="option-group">
|
||||
<input type="checkbox" id="optIp" checked>
|
||||
<label for="optIp">IP Intelligence</label>
|
||||
</div>
|
||||
<div class="option-group">
|
||||
<input type="checkbox" id="optHttp" checked>
|
||||
<label for="optHttp">HTTP Analysis</label>
|
||||
</div>
|
||||
<div class="option-group">
|
||||
<input type="checkbox" id="optBlacklist" checked>
|
||||
<label for="optBlacklist">Blacklists</label>
|
||||
</div>
|
||||
<div class="option-group slow">
|
||||
<input type="checkbox" id="optPorts">
|
||||
<label for="optPorts">Port Scan (lent)</label>
|
||||
</div>
|
||||
<div class="option-group slow">
|
||||
<input type="checkbox" id="optSubdomains">
|
||||
<label for="optSubdomains">Subdomenii (lent)</label>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="loading" id="loading">
|
||||
<div class="spinner"></div>
|
||||
<p>Se verifică domeniul... Acest proces poate dura până la 30 de secunde.</p>
|
||||
</div>
|
||||
|
||||
<div class="results" id="results"></div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const API_URL = '/api/v1/check/check';
|
||||
|
||||
document.getElementById('domainInput').addEventListener('keypress', (e) => {
|
||||
if (e.key === 'Enter') checkDomain();
|
||||
});
|
||||
|
||||
async function checkDomain() {
|
||||
const domain = document.getElementById('domainInput').value.trim();
|
||||
if (!domain) { alert('Introdu un domeniu!'); return; }
|
||||
|
||||
const btn = document.getElementById('checkBtn');
|
||||
const loading = document.getElementById('loading');
|
||||
const results = document.getElementById('results');
|
||||
|
||||
btn.disabled = true;
|
||||
loading.style.display = 'block';
|
||||
results.style.display = 'none';
|
||||
|
||||
const checkOptions = {
|
||||
whois: document.getElementById('optWhois').checked,
|
||||
dns: document.getElementById('optDns').checked,
|
||||
ssl: document.getElementById('optSsl').checked,
|
||||
ip_intelligence: document.getElementById('optIp').checked,
|
||||
http_analysis: document.getElementById('optHttp').checked,
|
||||
blacklist: document.getElementById('optBlacklist').checked,
|
||||
port_scan: document.getElementById('optPorts').checked,
|
||||
subdomains: document.getElementById('optSubdomains').checked
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await fetch(API_URL, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ domain, check_options: checkOptions })
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
displayResults(data.data, data.metadata);
|
||||
} else {
|
||||
results.innerHTML = `<div class="error">❌ ${data.error?.message || 'Eroare necunoscută'}</div>`;
|
||||
}
|
||||
} catch (err) {
|
||||
results.innerHTML = `<div class="error">❌ Eroare de conexiune: ${err.message}</div>`;
|
||||
}
|
||||
|
||||
btn.disabled = false;
|
||||
loading.style.display = 'none';
|
||||
results.style.display = 'block';
|
||||
}
|
||||
|
||||
function displayResults(d, meta) {
|
||||
let html = '<div class="results-grid">';
|
||||
if (d.availability) html += renderAvailabilityCard(d.availability);
|
||||
if (d.risk_score) html += renderRiskCard(d.risk_score);
|
||||
if (d.whois) html += renderWhoisCard(d.whois);
|
||||
if (d.dns) html += renderDnsCard(d.dns);
|
||||
if (d.mail) html += renderMailCard(d.mail);
|
||||
if (d.ssl) html += renderSslCard(d.ssl);
|
||||
if (d.ip_intelligence) html += renderIpCard(d.ip_intelligence);
|
||||
if (d.http_analysis) html += renderHttpCard(d.http_analysis);
|
||||
if (d.blacklist) html += renderBlacklistCard(d.blacklist);
|
||||
if (d.port_scan) html += renderPortCard(d.port_scan);
|
||||
if (d.subdomains) html += renderSubdomainsCard(d.subdomains);
|
||||
html += '</div>';
|
||||
html += `<div style="text-align: center; margin-top: 20px; color: #666; font-size: 0.85rem;">
|
||||
Check ID: ${d.check_id} | Timp procesare: ${meta.processing_time_ms}ms | API: ${meta.api_version}
|
||||
</div>`;
|
||||
document.getElementById('results').innerHTML = html;
|
||||
}
|
||||
|
||||
function renderAvailabilityCard(a) {
|
||||
const reg = a.is_registered;
|
||||
const label = reg ? '🔒 Înregistrat' : '🟢 Disponibil';
|
||||
const badgeClass = reg ? 'info' : 'success';
|
||||
return `
|
||||
<div class="card">
|
||||
<div class="card-header"><h3>📌 Disponibilitate</h3><span class="source">WHOIS+DNS</span></div>
|
||||
<div class="card-body">
|
||||
<div class="data-row"><div class="data-label">Status:</div><div class="data-value"><span class="badge ${badgeClass}" style="font-size:1rem">${label}</span></div></div>
|
||||
<div class="data-row"><div class="data-label">Încredere:</div><div class="data-value">${a.confidence}</div></div>
|
||||
<div class="data-row"><div class="data-label">Semnale:</div><div class="data-value">
|
||||
<span class="badge ${a.signals?.whois_has_data ? 'success' : 'neutral'}">WHOIS ${a.signals?.whois_has_data ? '✓' : '–'}</span>
|
||||
<span class="badge ${a.signals?.dns_resolves ? 'success' : 'neutral'}">DNS ${a.signals?.dns_resolves ? '✓' : '–'}</span>
|
||||
</div></div>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderMailCard(m) {
|
||||
const gradeColor = {A:'#00c853', B:'#8bc34a', C:'#ffc107', D:'#ff9800', F:'#f44336'}[m.mail_security_grade] || '#888';
|
||||
const spf = m.spf || {}, dmarc = m.dmarc || {}, dkim = m.dkim || {}, mx = m.mx || {}, sts = m.mta_sts || {};
|
||||
const tlsRow = (mx.starttls !== null && mx.starttls !== undefined) ? ` · STARTTLS ${mx.starttls ? '✓' : '✗'}` : '';
|
||||
return `
|
||||
<div class="card">
|
||||
<div class="card-header"><h3>📧 Mail Intelligence</h3><span class="source">EMAIL</span></div>
|
||||
<div class="card-body">
|
||||
<div class="data-row"><div class="data-label">Securitate email:</div><div class="data-value"><span class="badge" style="background:${gradeColor};color:#fff">${m.mail_security_grade} · ${m.mail_security_score}/100</span></div></div>
|
||||
${m.provider ? `<div class="data-row"><div class="data-label">Provider:</div><div class="data-value">${m.provider}</div></div>` : ''}
|
||||
<div class="data-row"><div class="data-label">MX:</div><div class="data-value">${mx.count || 0} servere${tlsRow}${mx.dane ? ' · DANE ✓' : ''}</div></div>
|
||||
<div class="data-row"><div class="data-label">SPF:</div><div class="data-value"><span class="badge ${spf.present ? 'success' : 'danger'}">${spf.present ? '✓' : '✗'}</span> ${spf.policy || ''}</div></div>
|
||||
<div class="data-row"><div class="data-label">DKIM:</div><div class="data-value"><span class="badge ${dkim.present ? 'success' : 'danger'}">${dkim.present ? '✓' : '✗'}</span> ${(dkim.selectors_found || []).join(', ')}</div></div>
|
||||
<div class="data-row"><div class="data-label">DMARC:</div><div class="data-value"><span class="badge ${dmarc.present ? 'success' : 'danger'}">${dmarc.present ? '✓' : '✗'}</span> ${dmarc.policy ? ('p=' + dmarc.policy) : ''}</div></div>
|
||||
<div class="data-row"><div class="data-label">MTA-STS:</div><div class="data-value"><span class="badge ${sts.present ? 'success' : 'neutral'}">${sts.present ? '✓' : '–'}</span>${sts.mode ? (' ' + sts.mode) : ''} ${m.tls_rpt?.present ? '<span class="badge success">TLS-RPT ✓</span>' : ''}</div></div>
|
||||
${m.deliverability?.tested ? `<div class="data-row"><div class="data-label">Catch-all:</div><div class="data-value">${m.deliverability.catch_all ? '<span class="badge warning">DA</span>' : '<span class="badge success">NU</span>'}</div></div>` : ''}
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderRiskCard(risk) {
|
||||
const levelClass = `risk-${risk.level.toLowerCase()}`;
|
||||
let factorsHtml = '';
|
||||
if (risk.factors && risk.factors.length > 0) {
|
||||
risk.factors.forEach(f => {
|
||||
const barColor = f.score <= 25 ? '#00c853' : f.score <= 50 ? '#ffc107' : f.score <= 75 ? '#ff9800' : '#f44336';
|
||||
factorsHtml += `
|
||||
<div class="risk-factor">
|
||||
<div class="name">${formatFactorName(f.factor)}</div>
|
||||
<div class="score">
|
||||
<div class="bar"><div class="bar-fill" style="width: ${f.score}%; background: ${barColor}"></div></div>
|
||||
<div class="value">${f.score}</div>
|
||||
</div>
|
||||
</div>`;
|
||||
});
|
||||
}
|
||||
let formulaHtml = '';
|
||||
if (risk.formula_breakdown && risk.formula_breakdown.length > 0) {
|
||||
formulaHtml = '<div class="formula-box"><h4>📐 Formula de Calcul</h4>';
|
||||
risk.formula_breakdown.forEach(fb => {
|
||||
formulaHtml += `<div class="formula-line">${fb.category}: ${fb.formula}</div>`;
|
||||
});
|
||||
formulaHtml += `<div class="formula-line" style="margin-top: 10px; font-weight: bold; color: #00d9ff;">${risk.formula_string || ''}</div></div>`;
|
||||
}
|
||||
return `
|
||||
<div class="card risk-card">
|
||||
<div class="card-header"><h3>🎯 Scor de Risc</h3><span class="source">CALCULATED</span></div>
|
||||
<div class="card-body">
|
||||
<div class="risk-score-display">
|
||||
<div class="risk-circle ${levelClass}">${risk.total}<span class="label">${risk.level}</span></div>
|
||||
<div class="risk-details">
|
||||
<div style="margin-bottom: 15px;">
|
||||
${risk.is_new_domain ? '<span class="badge warning">Domeniu Nou</span>' : ''}
|
||||
${risk.is_suspicious ? '<span class="badge danger">Suspect</span>' : ''}
|
||||
${risk.is_blacklisted ? '<span class="badge danger">Blacklisted</span>' : ''}
|
||||
${!risk.is_suspicious && !risk.is_blacklisted ? '<span class="badge success">OK</span>' : ''}
|
||||
</div>
|
||||
${factorsHtml}
|
||||
</div>
|
||||
</div>
|
||||
${formulaHtml}
|
||||
<div style="margin-top: 15px; font-size: 0.85rem; color: #888;">Praguri: LOW (0-25) | MEDIUM (26-50) | HIGH (51-75) | CRITICAL (76-100)</div>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderWhoisCard(w) {
|
||||
const statusArray = Array.isArray(w.status) ? w.status : (w.status ? [w.status] : []);
|
||||
const nsArray = Array.isArray(w.name_servers) ? w.name_servers : (w.name_servers ? [w.name_servers] : []);
|
||||
return `
|
||||
<div class="card">
|
||||
<div class="card-header"><h3>📋 WHOIS</h3><span class="source">${w.data_source || 'WHOIS'}</span></div>
|
||||
<div class="card-body">
|
||||
${w.creation_date ? `<div class="data-row"><div class="data-label">Data Creare:</div><div class="data-value">${formatDate(w.creation_date)}</div></div>` : ''}
|
||||
${w.age_days !== null && w.age_days !== undefined ? `<div class="data-row"><div class="data-label">Vârsta:</div><div class="data-value"><strong>${w.age_days} zile</strong> (${Math.floor(w.age_days/365)} ani)</div></div>` : ''}
|
||||
${w.expiration_date ? `<div class="data-row"><div class="data-label">Expirare:</div><div class="data-value">${formatDate(w.expiration_date)}</div></div>` : ''}
|
||||
${w.registrar ? `<div class="data-row"><div class="data-label">Registrar:</div><div class="data-value">${w.registrar}</div></div>` : ''}
|
||||
${w.dnssec ? `<div class="data-row"><div class="data-label">DNSSEC:</div><div class="data-value">${w.dnssec}</div></div>` : ''}
|
||||
${statusArray.length > 0 ? `<div class="data-row"><div class="data-label">Status:</div><div class="data-value">${statusArray.map(s => `<span class="badge info">${s}</span>`).join('')}</div></div>` : ''}
|
||||
${nsArray.length > 0 ? `<div class="data-row"><div class="data-label">Name Servers:</div><div class="data-value">${nsArray.map(ns => `<span class="badge neutral">${ns}</span>`).join('')}</div></div>` : ''}
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderDnsCard(dns) {
|
||||
return `
|
||||
<div class="card">
|
||||
<div class="card-header"><h3>🌐 DNS Records</h3><span class="source">DNS</span></div>
|
||||
<div class="card-body">
|
||||
${dns.a_records?.length ? `<div class="data-row"><div class="data-label">A (IPv4):</div><div class="data-value">${dns.a_records.map(r => `<span class="badge info">${r}</span>`).join('')}</div></div>` : ''}
|
||||
${dns.aaaa_records?.length ? `<div class="data-row"><div class="data-label">AAAA (IPv6):</div><div class="data-value">${dns.aaaa_records.map(r => `<span class="badge info">${r}</span>`).join('')}</div></div>` : ''}
|
||||
${dns.mx_records?.length ? `<div class="data-row"><div class="data-label">MX:</div><div class="data-value">${dns.mx_records.map(r => `<span class="badge neutral">${r.priority} ${r.host}</span>`).join('')}</div></div>` : ''}
|
||||
${dns.ns_records?.length ? `<div class="data-row"><div class="data-label">NS:</div><div class="data-value">${dns.ns_records.map(r => `<span class="badge neutral">${r}</span>`).join('')}</div></div>` : ''}
|
||||
<div class="data-row"><div class="data-label">SPF:</div><div class="data-value"><span class="badge ${dns.has_spf ? 'success' : 'danger'}">${dns.has_spf ? '✓' : '✗'} SPF</span> <span style="color:#888;font-size:0.8rem">(DKIM/DMARC: vezi cardul Mail Intelligence)</span></div></div>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderSslCard(ssl) {
|
||||
if (!ssl.has_ssl) {
|
||||
return `<div class="card"><div class="card-header"><h3>🔒 SSL/TLS</h3><span class="source">SSL</span></div><div class="card-body"><div class="error">❌ Nu are certificat SSL</div></div></div>`;
|
||||
}
|
||||
return `
|
||||
<div class="card">
|
||||
<div class="card-header"><h3>🔒 SSL/TLS</h3><span class="source">SSL</span></div>
|
||||
<div class="card-body">
|
||||
<div class="data-row"><div class="data-label">Status:</div><div class="data-value">
|
||||
<span class="badge ${ssl.is_valid ? 'success' : 'danger'}">${ssl.is_valid ? '✓ Valid' : '✗ Invalid'}</span>
|
||||
${ssl.is_expired ? '<span class="badge danger">Expirat</span>' : ''}
|
||||
${ssl.is_wildcard ? '<span class="badge info">Wildcard</span>' : ''}
|
||||
</div></div>
|
||||
${ssl.issuer ? `<div class="data-row"><div class="data-label">Issuer:</div><div class="data-value">${ssl.issuer}</div></div>` : ''}
|
||||
${ssl.valid_until ? `<div class="data-row"><div class="data-label">Valid Until:</div><div class="data-value">${formatDate(ssl.valid_until)}</div></div>` : ''}
|
||||
${ssl.days_until_expiry !== null ? `<div class="data-row"><div class="data-label">Expiră în:</div><div class="data-value"><span class="badge ${ssl.days_until_expiry < 30 ? 'warning' : 'success'}">${ssl.days_until_expiry} zile</span></div></div>` : ''}
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderIpCard(ip) {
|
||||
return `
|
||||
<div class="card">
|
||||
<div class="card-header"><h3>🌍 IP Intelligence</h3><span class="source">ipinfo.io</span></div>
|
||||
<div class="card-body">
|
||||
<div class="data-row"><div class="data-label">IP:</div><div class="data-value"><strong>${ip.ip}</strong></div></div>
|
||||
${ip.reverse_dns ? `<div class="data-row"><div class="data-label">Reverse DNS:</div><div class="data-value">${ip.reverse_dns}</div></div>` : ''}
|
||||
${ip.country ? `<div class="data-row"><div class="data-label">Locație:</div><div class="data-value">${ip.city || ''} ${ip.region ? ', ' + ip.region : ''}, ${ip.country}</div></div>` : ''}
|
||||
${ip.asn ? `<div class="data-row"><div class="data-label">ASN:</div><div class="data-value">${ip.asn}</div></div>` : ''}
|
||||
${ip.isp ? `<div class="data-row"><div class="data-label">ISP:</div><div class="data-value">${ip.isp}</div></div>` : ''}
|
||||
<div class="data-row"><div class="data-label">Tip:</div><div class="data-value"><span class="badge ${ip.is_datacenter ? 'info' : 'success'}">${ip.is_datacenter ? 'Datacenter' : 'Rezidențial'}</span></div></div>
|
||||
${ip.hosting_score ? `<div class="data-row"><div class="data-label">Hosting Score:</div><div class="data-value"><span class="badge ${ip.hosting_score.score <= 30 ? 'success' : ip.hosting_score.score <= 50 ? 'warning' : 'danger'}">${ip.hosting_score.score}/100</span></div></div>` : ''}
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderHttpCard(http) {
|
||||
const secHeaders = http.security_headers || {};
|
||||
let headersHtml = '';
|
||||
['Strict-Transport-Security', 'X-Frame-Options', 'X-Content-Type-Options', 'Content-Security-Policy'].forEach(h => {
|
||||
headersHtml += `<div class="security-header"><span>${h}</span><span class="${secHeaders[h] ? 'header-present' : 'header-missing'}">${secHeaders[h] ? '✓' : '✗'}</span></div>`;
|
||||
});
|
||||
return `
|
||||
<div class="card">
|
||||
<div class="card-header"><h3>🌐 HTTP Analysis</h3><span class="source">HTTP</span></div>
|
||||
<div class="card-body">
|
||||
<div class="data-row"><div class="data-label">HTTPS:</div><div class="data-value">
|
||||
<span class="badge ${http.has_https ? 'success' : 'danger'}">${http.has_https ? '✓' : '✗'} HTTPS</span>
|
||||
${http.http_to_https_redirect ? '<span class="badge success">Auto-redirect</span>' : ''}
|
||||
</div></div>
|
||||
${http.server ? `<div class="data-row"><div class="data-label">Server:</div><div class="data-value">${http.server}</div></div>` : ''}
|
||||
${http.cms ? `<div class="data-row"><div class="data-label">CMS:</div><div class="data-value"><span class="badge info">${http.cms}</span></div></div>` : ''}
|
||||
${http.technologies?.length ? `<div class="data-row"><div class="data-label">Tech:</div><div class="data-value"><div class="tech-tags">${http.technologies.slice(0,8).map(t => `<span class="tech-tag">${t}</span>`).join('')}</div></div></div>` : ''}
|
||||
${http.response_time_ms ? `<div class="data-row"><div class="data-label">Response:</div><div class="data-value">${http.response_time_ms}ms</div></div>` : ''}
|
||||
<div style="margin-top: 15px; font-weight: 500; margin-bottom: 10px;">Security Headers:</div>
|
||||
${headersHtml}
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderBlacklistCard(bl) {
|
||||
let checksHtml = '';
|
||||
if (bl.ip_check) {
|
||||
const clean = bl.ip_check.clean_lists?.slice(0,4) || [];
|
||||
const listed = bl.ip_check.listings || [];
|
||||
checksHtml += `<div style="margin-bottom: 10px;"><strong>IP (${bl.ip_check.total_checked}):</strong> ${listed.map(l => `<span class="badge danger">${l.list_name}</span>`).join('')} ${clean.map(l => `<span class="badge success">${l}</span>`).join('')}</div>`;
|
||||
}
|
||||
if (bl.domain_check) {
|
||||
const clean = bl.domain_check.clean_lists?.slice(0,4) || [];
|
||||
const listed = bl.domain_check.listings || [];
|
||||
checksHtml += `<div><strong>Domain (${bl.domain_check.total_checked}):</strong> ${listed.map(l => `<span class="badge danger">${l.list_name}</span>`).join('')} ${clean.map(l => `<span class="badge success">${l}</span>`).join('')}</div>`;
|
||||
}
|
||||
return `
|
||||
<div class="card">
|
||||
<div class="card-header"><h3>🛡️ Blacklists</h3><span class="source">DNSBL</span></div>
|
||||
<div class="card-body">
|
||||
<div class="data-row"><div class="data-label">Status:</div><div class="data-value"><span class="badge ${bl.is_blacklisted ? 'danger' : 'success'}">${bl.is_blacklisted ? '❌ BLACKLISTED' : '✓ CLEAN'}</span></div></div>
|
||||
<div class="data-row"><div class="data-label">Reputation:</div><div class="data-value"><span class="badge ${bl.reputation_score >= 80 ? 'success' : bl.reputation_score >= 60 ? 'warning' : 'danger'}">${bl.reputation_score}/100</span></div></div>
|
||||
${checksHtml}
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderPortCard(ports) {
|
||||
const openPorts = ports.open_ports || [];
|
||||
const dangerous = ports.dangerous_open || [];
|
||||
const summary = ports.scan_summary || {};
|
||||
let portsHtml = '<div class="port-list">';
|
||||
openPorts.forEach(p => {
|
||||
const isDanger = dangerous.some(d => d.port === p.port);
|
||||
portsHtml += `<span class="port-item ${isDanger ? 'danger' : 'safe'}">${p.port} ${p.service}</span>`;
|
||||
});
|
||||
portsHtml += '</div>';
|
||||
return `
|
||||
<div class="card">
|
||||
<div class="card-header"><h3>🔌 Port Scan</h3><span class="source">TCP</span></div>
|
||||
<div class="card-body">
|
||||
<div class="data-row"><div class="data-label">Rezumat:</div><div class="data-value">
|
||||
<span class="badge info">${summary.open_count || 0} deschise</span>
|
||||
${summary.dangerous_count > 0 ? `<span class="badge danger">${summary.dangerous_count} periculoase</span>` : ''}
|
||||
</div></div>
|
||||
<div class="data-row"><div class="data-label">Porturi:</div><div class="data-value">${portsHtml}</div></div>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderSubdomainsCard(subs) {
|
||||
const subList = subs.subdomains || [];
|
||||
const src = subs.sources || {};
|
||||
const srcCounts = [
|
||||
['CT', (src.certificate_transparency || []).length],
|
||||
['Brute', (src.dns_bruteforce || []).length],
|
||||
['AXFR', (src.zone_transfer || []).length],
|
||||
['DNS', (src.dns_records || []).length],
|
||||
['Manual', (src.user_provided || []).length],
|
||||
].filter(([, n]) => n > 0).map(([k, n]) => `<span class="badge neutral">${k}: ${n}</span>`).join(' ');
|
||||
return `
|
||||
<div class="card">
|
||||
<div class="card-header"><h3>🔎 Subdomenii</h3><span class="source">CT+DNS+Brute</span></div>
|
||||
<div class="card-body">
|
||||
<div class="data-row"><div class="data-label">Total:</div><div class="data-value"><strong>${subs.total_found}</strong> ${srcCounts}</div></div>
|
||||
<div class="data-row"><div class="data-label">Lista:</div><div class="data-value"><div class="tech-tags">${subList.slice(0, 20).map(s => `<span class="tech-tag">${s}</span>`).join('')}${subList.length > 20 ? `<span class="badge neutral">+${subList.length - 20}</span>` : ''}</div></div></div>
|
||||
${subs.has_wildcard_cert ? `<div class="data-row"><div class="data-label">⚠️ Wildcard:</div><div class="data-value"><span class="badge warning">cert *.${subs.domain}</span></div></div>` : ''}
|
||||
${subs.coverage_note ? `<div style="margin-top:8px;font-size:0.78rem;color:#b8860b;line-height:1.4">ℹ️ ${subs.coverage_note}</div>` : ''}
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function formatFactorName(factor) {
|
||||
const names = {'domain_age': '📅 Vârsta Domeniu', 'ssl': '🔒 SSL/TLS', 'dns': '🌐 DNS', 'email_security': '📧 Email Security', 'whois': '📋 WHOIS', 'ip_reputation': '🌍 IP Reputation', 'http_security': '🔐 HTTP Security', 'blacklist': '🛡️ Blacklists', 'port_security': '🔌 Porturi'};
|
||||
return names[factor] || factor;
|
||||
}
|
||||
|
||||
function formatDate(dateStr) {
|
||||
if (!dateStr) return '-';
|
||||
try { return new Date(dateStr).toLocaleDateString('ro-RO', { year: 'numeric', month: 'long', day: 'numeric' }); }
|
||||
catch { return dateStr; }
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
91
ai_platform/modules/domain_check/api/requirements.txt
Normal file
91
ai_platform/modules/domain_check/api/requirements.txt
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
# Flask Framework
|
||||
Flask==3.0.0
|
||||
Flask-RESTX==1.3.0
|
||||
Flask-SQLAlchemy==3.1.1
|
||||
Flask-Migrate==4.0.5
|
||||
Flask-CORS==4.0.0
|
||||
Flask-Limiter==3.5.0
|
||||
|
||||
# Database
|
||||
SQLAlchemy==2.0.23
|
||||
psycopg2-binary==2.9.9
|
||||
alembic==1.13.1
|
||||
|
||||
# Redis & Caching
|
||||
redis==5.0.1
|
||||
hiredis==2.3.2
|
||||
|
||||
# Celery (Task Queue)
|
||||
celery==5.3.4
|
||||
|
||||
# HTTP Requests (moved before WHOIS tools to resolve dependencies)
|
||||
requests==2.31.0
|
||||
urllib3==2.1.0
|
||||
httpx==0.25.2
|
||||
|
||||
# WHOIS & Domain Tools
|
||||
whoisit>=2.0.0
|
||||
python-whois==0.8.0
|
||||
dnspython==2.4.2
|
||||
tldextract==5.1.1
|
||||
|
||||
# Validation & Serialization
|
||||
marshmallow==3.20.1
|
||||
marshmallow-sqlalchemy==0.29.0
|
||||
pydantic==2.5.2
|
||||
|
||||
# Environment & Configuration
|
||||
python-dotenv==1.0.0
|
||||
environs==10.3.0
|
||||
|
||||
# Date & Time
|
||||
python-dateutil==2.8.2
|
||||
pytz==2023.3
|
||||
|
||||
# Logging & Monitoring
|
||||
structlog==23.3.0
|
||||
sentry-sdk[flask]==1.39.1
|
||||
|
||||
# Security
|
||||
PyJWT==2.8.0
|
||||
cryptography==41.0.7
|
||||
validators==0.22.0
|
||||
|
||||
# API Documentation
|
||||
apispec==6.3.1
|
||||
apispec-webframeworks==1.0.0
|
||||
|
||||
# Utilities
|
||||
Click==8.1.7
|
||||
colorama==0.4.6
|
||||
python-slugify==8.0.1
|
||||
shortuuid==1.0.11
|
||||
|
||||
# Testing
|
||||
pytest==7.4.3
|
||||
pytest-cov==4.1.0
|
||||
pytest-flask==1.3.0
|
||||
pytest-mock==3.12.0
|
||||
faker==21.0.0
|
||||
|
||||
# Code Quality
|
||||
black==23.12.1
|
||||
flake8==6.1.0
|
||||
pylint==3.0.3
|
||||
mypy==1.7.1
|
||||
|
||||
# Performance
|
||||
gunicorn==21.2.0
|
||||
gevent==23.9.1
|
||||
|
||||
# Data Processing
|
||||
pandas==2.1.4
|
||||
numpy==1.26.2
|
||||
|
||||
# IP & Network
|
||||
ipaddress==1.0.23
|
||||
geoip2==4.7.0
|
||||
|
||||
# SSL Certificate Handling
|
||||
certifi==2023.11.17
|
||||
pyOpenSSL==23.3.0
|
||||
45
ai_platform/modules/domain_check/api/run.py
Normal file
45
ai_platform/modules/domain_check/api/run.py
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
"""
|
||||
Domain Check API - Entry Point
|
||||
Version: 1.0.0
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add app directory to Python path
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from app import create_app
|
||||
|
||||
# Create Flask app instance
|
||||
app = create_app(os.getenv('FLASK_ENV', 'development'))
|
||||
|
||||
if __name__ == '__main__':
|
||||
host = os.getenv('API_HOST', '0.0.0.0')
|
||||
port = int(os.getenv('API_PORT', 5000))
|
||||
debug = os.getenv('DEBUG', 'True').lower() == 'true'
|
||||
|
||||
print(f"""
|
||||
╔══════════════════════════════════════════╗
|
||||
║ Domain Check API - Anti-Fake News ║
|
||||
║ Version: 1.0.0 ║
|
||||
║ Environment: {os.getenv('FLASK_ENV', 'development'):<26}║
|
||||
║ Host: {host:<32}║
|
||||
║ Port: {port:<32}║
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
📚 API Documentation:
|
||||
- Swagger UI: http://{host}:{port}/docs
|
||||
- ReDoc: http://{host}:{port}/redoc
|
||||
|
||||
🏥 Health Check: http://{host}:{port}/health
|
||||
|
||||
🚀 Starting server...
|
||||
""")
|
||||
|
||||
app.run(
|
||||
host=host,
|
||||
port=port,
|
||||
debug=debug,
|
||||
threaded=True
|
||||
)
|
||||
104
ai_platform/modules/domain_check/api/tests/test_whois_dates.py
Normal file
104
ai_platform/modules/domain_check/api/tests/test_whois_dates.py
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
"""
|
||||
Regression tests for WHOIS date handling.
|
||||
|
||||
Root cause of the historical HTTP 500 on `.ro` domains (e.g. exemplu.ro):
|
||||
python-whois returns ROTLD dates as raw strings, and downstream code did
|
||||
`datetime - str`, raising TypeError. These tests lock in the fix.
|
||||
"""
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.whois_service import normalize_whois_date, _as_list, WhoisService
|
||||
from app.services.risk_scorer import RiskScorer, _coerce_datetime
|
||||
|
||||
|
||||
# --- normalize_whois_date -------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("value", [
|
||||
None,
|
||||
"",
|
||||
" ",
|
||||
"none",
|
||||
"N/A",
|
||||
])
|
||||
def test_normalize_returns_none_for_empty(value):
|
||||
assert normalize_whois_date(value) is None
|
||||
|
||||
|
||||
def test_normalize_passthrough_datetime():
|
||||
dt = datetime(2020, 1, 2, 3, 4, 5)
|
||||
assert normalize_whois_date(dt) == dt
|
||||
|
||||
|
||||
def test_normalize_iso_string():
|
||||
assert normalize_whois_date("2020-01-02T03:04:05Z") == datetime(2020, 1, 2, 3, 4, 5)
|
||||
|
||||
|
||||
def test_normalize_rotld_style_string():
|
||||
# ROTLD / free-form formats that broke datetime.fromisoformat
|
||||
assert normalize_whois_date("2015-03-22") == datetime(2015, 3, 22)
|
||||
assert normalize_whois_date("22.03.2015").year == 2015
|
||||
assert normalize_whois_date("Before 2012-08-10") is not None # fuzzy
|
||||
|
||||
|
||||
def test_normalize_list_takes_first_valid():
|
||||
out = normalize_whois_date([None, "2019-05-06", datetime(2021, 1, 1)])
|
||||
assert out == datetime(2019, 5, 6)
|
||||
|
||||
|
||||
def test_normalize_garbage_returns_none():
|
||||
assert normalize_whois_date("not a date at all !!") is None
|
||||
|
||||
|
||||
def test_as_list_normalizes():
|
||||
assert _as_list(None) == []
|
||||
assert _as_list("a") == ["a"]
|
||||
assert _as_list(["a", None, "", "b"]) == ["a", "b"]
|
||||
|
||||
|
||||
# --- the actual crash site: risk scorer must never raise on bad dates -----
|
||||
|
||||
def test_score_domain_age_does_not_raise_on_string():
|
||||
"""The exact regression: a raw string creation_date must not 500."""
|
||||
scorer = RiskScorer()
|
||||
score, details = scorer._score_domain_age({'creation_date': '2015-03-22'})
|
||||
assert isinstance(score, int)
|
||||
assert details['details']['age_days'] is not None
|
||||
|
||||
|
||||
def test_score_domain_age_does_not_raise_on_garbage():
|
||||
scorer = RiskScorer()
|
||||
score, details = scorer._score_domain_age({'creation_date': 'totally-not-a-date'})
|
||||
# Garbage coerces to None -> "unable to determine" path, never an exception.
|
||||
assert score == 50
|
||||
assert details['details']['age_days'] is None
|
||||
|
||||
|
||||
def test_score_domain_age_missing():
|
||||
scorer = RiskScorer()
|
||||
score, details = scorer._score_domain_age({})
|
||||
assert score == 50
|
||||
|
||||
|
||||
def test_coerce_datetime():
|
||||
assert _coerce_datetime(None) is None
|
||||
assert _coerce_datetime("2020-01-01") == datetime(2020, 1, 1)
|
||||
assert _coerce_datetime(datetime(2020, 1, 1)) == datetime(2020, 1, 1)
|
||||
assert _coerce_datetime("garbage") is None
|
||||
|
||||
|
||||
# --- is_registered signal -------------------------------------------------
|
||||
|
||||
def test_is_registered_true_when_creation_date():
|
||||
assert WhoisService._is_registered({'creation_date': datetime(2020, 1, 1)}) is True
|
||||
|
||||
|
||||
def test_is_registered_true_when_registrar():
|
||||
assert WhoisService._is_registered({'registrar': 'ROTLD'}) is True
|
||||
|
||||
|
||||
def test_is_registered_none_when_sparse():
|
||||
# No positive signal -> inconclusive (None), DNS decides downstream.
|
||||
assert WhoisService._is_registered({'creation_date': None, 'registrar': None,
|
||||
'name_servers': [], 'status': []}) is None
|
||||
Loading…
Add table
Add a link
Reference in a new issue