- Add AI moderation to comments (was missing protection) - Enhance post moderation to analyze images, videos, thumbnails - Add FlagComment method for comment flagging - Extract media URLs for comprehensive content analysis - Update moderation config and models - Add OpenAI and Google Vision API integration - Fix database connection to use localhost This ensures all text, image, and video content is protected by AI moderation.
504 lines
14 KiB
Go
504 lines
14 KiB
Go
package services
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
type ModerationService struct {
|
|
pool *pgxpool.Pool
|
|
httpClient *http.Client
|
|
openAIKey string
|
|
googleKey string
|
|
}
|
|
|
|
func NewModerationService(pool *pgxpool.Pool, openAIKey, googleKey string) *ModerationService {
|
|
return &ModerationService{
|
|
pool: pool,
|
|
httpClient: &http.Client{
|
|
Timeout: 30 * time.Second,
|
|
},
|
|
openAIKey: openAIKey,
|
|
googleKey: googleKey,
|
|
}
|
|
}
|
|
|
|
type ThreePoisonsScore struct {
|
|
Hate float64 `json:"hate"`
|
|
Greed float64 `json:"greed"`
|
|
Delusion float64 `json:"delusion"`
|
|
}
|
|
|
|
// OpenAIModerationResponse represents the response from OpenAI Moderation API
|
|
type OpenAIModerationResponse struct {
|
|
Results []struct {
|
|
Categories struct {
|
|
Hate float64 `json:"hate"`
|
|
HateThreatening float64 `json:"hate/threatening"`
|
|
SelfHarm float64 `json:"self-harm"`
|
|
SelfHarmIntent float64 `json:"self-harm/intent"`
|
|
SelfHarmInstructions float64 `json:"self-harm/instructions"`
|
|
Sexual float64 `json:"sexual"`
|
|
SexualMinors float64 `json:"sexual/minors"`
|
|
Violence float64 `json:"violence"`
|
|
ViolenceGraphic float64 `json:"violence/graphic"`
|
|
} `json:"categories"`
|
|
CategoryScores struct {
|
|
Hate float64 `json:"hate"`
|
|
HateThreatening float64 `json:"hate/threatening"`
|
|
SelfHarm float64 `json:"self-harm"`
|
|
SelfHarmIntent float64 `json:"self-harm/intent"`
|
|
SelfHarmInstructions float64 `json:"self-harm/instructions"`
|
|
Sexual float64 `json:"sexual"`
|
|
SexualMinors float64 `json:"sexual/minors"`
|
|
Violence float64 `json:"violence"`
|
|
ViolenceGraphic float64 `json:"violence/graphic"`
|
|
} `json:"category_scores"`
|
|
Flagged bool `json:"flagged"`
|
|
} `json:"results"`
|
|
}
|
|
|
|
// GoogleVisionSafeSearch represents SafeSearch detection results
|
|
type GoogleVisionSafeSearch struct {
|
|
Adult string `json:"adult"`
|
|
Spoof string `json:"spoof"`
|
|
Medical string `json:"medical"`
|
|
Violence string `json:"violence"`
|
|
Racy string `json:"racy"`
|
|
}
|
|
|
|
// GoogleVisionResponse represents the response from Google Vision API
|
|
type GoogleVisionResponse struct {
|
|
Responses []struct {
|
|
SafeSearchAnnotation GoogleVisionSafeSearch `json:"safeSearchAnnotation"`
|
|
} `json:"responses"`
|
|
}
|
|
|
|
func (s *ModerationService) AnalyzeContent(ctx context.Context, body string, mediaURLs []string) (*ThreePoisonsScore, string, error) {
|
|
score := &ThreePoisonsScore{
|
|
Hate: 0.0,
|
|
Greed: 0.0,
|
|
Delusion: 0.0,
|
|
}
|
|
|
|
// Analyze text with OpenAI Moderation API
|
|
if s.openAIKey != "" && body != "" {
|
|
openAIScore, err := s.analyzeWithOpenAI(ctx, body)
|
|
if err != nil {
|
|
// Log error but continue with fallback
|
|
fmt.Printf("OpenAI moderation error: %v\n", err)
|
|
} else {
|
|
score = openAIScore
|
|
}
|
|
}
|
|
|
|
// Analyze media with Google Vision API if provided
|
|
if s.googleKey != "" && len(mediaURLs) > 0 {
|
|
visionScore, err := s.analyzeWithGoogleVision(ctx, mediaURLs)
|
|
if err != nil {
|
|
fmt.Printf("Google Vision analysis error: %v\n", err)
|
|
} else {
|
|
// Merge vision scores with existing scores
|
|
if visionScore.Hate > score.Hate {
|
|
score.Hate = visionScore.Hate
|
|
}
|
|
if visionScore.Delusion > score.Delusion {
|
|
score.Delusion = visionScore.Delusion
|
|
}
|
|
}
|
|
}
|
|
|
|
// Fallback to keyword-based analysis for greed/spam detection
|
|
if score.Greed == 0.0 {
|
|
greedKeywords := []string{
|
|
"buy", "crypto", "rich", "scam", "investment", "profit",
|
|
"money", "cash", "bitcoin", "ethereum", "trading", "forex",
|
|
"get rich", "quick money", "guaranteed returns", "multiplier",
|
|
}
|
|
if containsAny(body, greedKeywords) {
|
|
score.Greed = 0.7
|
|
}
|
|
}
|
|
|
|
// Determine primary flag reason
|
|
flagReason := ""
|
|
if score.Hate > 0.5 {
|
|
flagReason = "hate"
|
|
} else if score.Greed > 0.5 {
|
|
flagReason = "greed"
|
|
} else if score.Delusion > 0.5 {
|
|
flagReason = "delusion"
|
|
}
|
|
|
|
return score, flagReason, nil
|
|
}
|
|
|
|
// analyzeWithOpenAI sends content to OpenAI Moderation API
|
|
func (s *ModerationService) analyzeWithOpenAI(ctx context.Context, content string) (*ThreePoisonsScore, error) {
|
|
if s.openAIKey == "" {
|
|
return nil, fmt.Errorf("OpenAI API key not configured")
|
|
}
|
|
|
|
requestBody := map[string]interface{}{
|
|
"input": content,
|
|
"model": "text-moderation-latest",
|
|
}
|
|
|
|
jsonBody, err := json.Marshal(requestBody)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to marshal request: %w", err)
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.openai.com/v1/moderations", bytes.NewBuffer(jsonBody))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create request: %w", err)
|
|
}
|
|
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("Authorization", "Bearer "+s.openAIKey)
|
|
|
|
resp, err := s.httpClient.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to send request: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
body, _ := io.ReadAll(resp.Body)
|
|
return nil, fmt.Errorf("OpenAI API error: %d - %s", resp.StatusCode, string(body))
|
|
}
|
|
|
|
var moderationResp OpenAIModerationResponse
|
|
if err := json.NewDecoder(resp.Body).Decode(&moderationResp); err != nil {
|
|
return nil, fmt.Errorf("failed to decode response: %w", err)
|
|
}
|
|
|
|
if len(moderationResp.Results) == 0 {
|
|
return &ThreePoisonsScore{Hate: 0, Greed: 0, Delusion: 0}, nil
|
|
}
|
|
|
|
result := moderationResp.Results[0]
|
|
score := &ThreePoisonsScore{
|
|
// Map OpenAI categories to Three Poisons
|
|
Hate: max(
|
|
result.Categories.Hate,
|
|
result.Categories.HateThreatening,
|
|
result.Categories.Violence,
|
|
result.Categories.ViolenceGraphic,
|
|
),
|
|
Greed: 0, // OpenAI doesn't detect greed/spam well
|
|
Delusion: max(
|
|
result.Categories.SelfHarm,
|
|
result.Categories.SelfHarmIntent,
|
|
result.Categories.SelfHarmInstructions,
|
|
),
|
|
}
|
|
|
|
return score, nil
|
|
}
|
|
|
|
// analyzeWithGoogleVision analyzes images for inappropriate content
|
|
func (s *ModerationService) analyzeWithGoogleVision(ctx context.Context, mediaURLs []string) (*ThreePoisonsScore, error) {
|
|
if s.googleKey == "" {
|
|
return nil, fmt.Errorf("Google Vision API key not configured")
|
|
}
|
|
|
|
score := &ThreePoisonsScore{
|
|
Hate: 0.0,
|
|
Greed: 0.0,
|
|
Delusion: 0.0,
|
|
}
|
|
|
|
for _, mediaURL := range mediaURLs {
|
|
// Only process image URLs
|
|
if !isImageURL(mediaURL) {
|
|
continue
|
|
}
|
|
|
|
requestBody := map[string]interface{}{
|
|
"requests": []map[string]interface{}{
|
|
{
|
|
"image": map[string]interface{}{
|
|
"source": map[string]interface{}{
|
|
"imageUri": mediaURL,
|
|
},
|
|
},
|
|
"features": []map[string]interface{}{
|
|
{
|
|
"type": "SAFE_SEARCH_DETECTION",
|
|
"maxResults": 1,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
jsonBody, err := json.Marshal(requestBody)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(ctx, "POST", "https://vision.googleapis.com/v1/images:annotate", bytes.NewBuffer(jsonBody))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.URL.RawQuery = "key=" + s.googleKey
|
|
|
|
resp, err := s.httpClient.Do(req)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
continue
|
|
}
|
|
|
|
var visionResp GoogleVisionResponse
|
|
if err := json.NewDecoder(resp.Body).Decode(&visionResp); err != nil {
|
|
continue
|
|
}
|
|
|
|
if len(visionResp.Responses) > 0 {
|
|
imageScore := s.convertVisionScore(visionResp.Responses[0].SafeSearchAnnotation)
|
|
// Merge with overall score (take maximum)
|
|
if imageScore.Hate > score.Hate {
|
|
score.Hate = imageScore.Hate
|
|
}
|
|
if imageScore.Delusion > score.Delusion {
|
|
score.Delusion = imageScore.Delusion
|
|
}
|
|
}
|
|
}
|
|
|
|
return score, nil
|
|
}
|
|
|
|
// convertVisionScore converts Google Vision SafeSearch results to ThreePoisonsScore
|
|
func (s *ModerationService) convertVisionScore(safeSearch GoogleVisionSafeSearch) *ThreePoisonsScore {
|
|
score := &ThreePoisonsScore{
|
|
Hate: 0.0,
|
|
Greed: 0.0,
|
|
Delusion: 0.0,
|
|
}
|
|
|
|
// Convert string likelihoods to numeric scores
|
|
likelihoodToScore := map[string]float64{
|
|
"UNKNOWN": 0.0,
|
|
"VERY_UNLIKELY": 0.1,
|
|
"UNLIKELY": 0.3,
|
|
"POSSIBLE": 0.5,
|
|
"LIKELY": 0.7,
|
|
"VERY_LIKELY": 0.9,
|
|
}
|
|
|
|
// Map Vision categories to Three Poisons
|
|
if hateScore, ok := likelihoodToScore[safeSearch.Violence]; ok {
|
|
score.Hate = hateScore
|
|
}
|
|
if adultScore, ok := likelihoodToScore[safeSearch.Adult]; ok && adultScore > score.Hate {
|
|
score.Hate = adultScore
|
|
}
|
|
if racyScore, ok := likelihoodToScore[safeSearch.Racy]; ok && racyScore > score.Delusion {
|
|
score.Delusion = racyScore
|
|
}
|
|
|
|
return score
|
|
}
|
|
|
|
// Helper function to get maximum of multiple floats
|
|
func max(values ...float64) float64 {
|
|
maxVal := 0.0
|
|
for _, v := range values {
|
|
if v > maxVal {
|
|
maxVal = v
|
|
}
|
|
}
|
|
return maxVal
|
|
}
|
|
|
|
// Helper function to check if URL is an image
|
|
func isImageURL(url string) bool {
|
|
imageExtensions := []string{".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp"}
|
|
lowerURL := strings.ToLower(url)
|
|
for _, ext := range imageExtensions {
|
|
if strings.HasSuffix(lowerURL, ext) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (s *ModerationService) FlagPost(ctx context.Context, postID uuid.UUID, scores *ThreePoisonsScore, reason string) error {
|
|
scoresJSON, err := json.Marshal(scores)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to marshal scores: %w", err)
|
|
}
|
|
|
|
query := `
|
|
INSERT INTO moderation_flags (post_id, flag_reason, scores, status)
|
|
VALUES ($1, $2, $3, 'pending')
|
|
RETURNING id, created_at
|
|
`
|
|
|
|
var flagID uuid.UUID
|
|
var createdAt time.Time
|
|
err = s.pool.QueryRow(ctx, query, postID, reason, scoresJSON).Scan(&flagID, &createdAt)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to insert moderation flag: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *ModerationService) FlagComment(ctx context.Context, commentID uuid.UUID, scores *ThreePoisonsScore, reason string) error {
|
|
scoresJSON, err := json.Marshal(scores)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to marshal scores: %w", err)
|
|
}
|
|
|
|
query := `
|
|
INSERT INTO moderation_flags (comment_id, flag_reason, scores, status)
|
|
VALUES ($1, $2, $3, 'pending')
|
|
RETURNING id, created_at
|
|
`
|
|
|
|
var flagID uuid.UUID
|
|
var createdAt time.Time
|
|
err = s.pool.QueryRow(ctx, query, commentID, reason, scoresJSON).Scan(&flagID, &createdAt)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to insert comment moderation flag: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// GetPendingFlags retrieves all pending moderation flags
|
|
func (s *ModerationService) GetPendingFlags(ctx context.Context, limit, offset int) ([]map[string]interface{}, error) {
|
|
query := `
|
|
SELECT
|
|
mf.id, mf.post_id, mf.comment_id, mf.flag_reason, mf.scores,
|
|
mf.status, mf.created_at,
|
|
p.content as post_content,
|
|
c.content as comment_content,
|
|
u.handle as author_handle
|
|
FROM moderation_flags mf
|
|
LEFT JOIN posts p ON mf.post_id = p.id
|
|
LEFT JOIN comments c ON mf.comment_id = c.id
|
|
LEFT JOIN users u ON (p.user_id = u.id OR c.user_id = u.id)
|
|
WHERE mf.status = 'pending'
|
|
ORDER BY mf.created_at ASC
|
|
LIMIT $1 OFFSET $2
|
|
`
|
|
|
|
rows, err := s.pool.Query(ctx, query, limit, offset)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to query pending flags: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
|
|
var flags []map[string]interface{}
|
|
for rows.Next() {
|
|
var id, postID, commentID uuid.UUID
|
|
var flagReason, status string
|
|
var scoresJSON []byte
|
|
var createdAt time.Time
|
|
var postContent, commentContent, authorHandle *string
|
|
|
|
err := rows.Scan(&id, &postID, &commentID, &flagReason, &scoresJSON, &status, &createdAt, &postContent, &commentContent, &authorHandle)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to scan flag row: %w", err)
|
|
}
|
|
|
|
var scores map[string]float64
|
|
if err := json.Unmarshal(scoresJSON, &scores); err != nil {
|
|
return nil, fmt.Errorf("failed to unmarshal scores: %w", err)
|
|
}
|
|
|
|
flag := map[string]interface{}{
|
|
"id": id,
|
|
"post_id": postID,
|
|
"comment_id": commentID,
|
|
"flag_reason": flagReason,
|
|
"scores": scores,
|
|
"status": status,
|
|
"created_at": createdAt,
|
|
"post_content": postContent,
|
|
"comment_content": commentContent,
|
|
"author_handle": authorHandle,
|
|
}
|
|
|
|
flags = append(flags, flag)
|
|
}
|
|
|
|
return flags, nil
|
|
}
|
|
|
|
// UpdateFlagStatus updates the status of a moderation flag
|
|
func (s *ModerationService) UpdateFlagStatus(ctx context.Context, flagID uuid.UUID, status string, reviewedBy uuid.UUID) error {
|
|
query := `
|
|
UPDATE moderation_flags
|
|
SET status = $1, reviewed_by = $2, reviewed_at = NOW()
|
|
WHERE id = $3
|
|
`
|
|
|
|
_, err := s.pool.Exec(ctx, query, status, reviewedBy, flagID)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to update flag status: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// UpdateUserStatus updates a user's moderation status
|
|
func (s *ModerationService) UpdateUserStatus(ctx context.Context, userID uuid.UUID, status string, changedBy uuid.UUID, reason string) error {
|
|
query := `
|
|
UPDATE users
|
|
SET status = $1
|
|
WHERE id = $2
|
|
`
|
|
|
|
_, err := s.pool.Exec(ctx, query, status, userID)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to update user status: %w", err)
|
|
}
|
|
|
|
// Log the status change
|
|
historyQuery := `
|
|
INSERT INTO user_status_history (user_id, old_status, new_status, reason, changed_by)
|
|
SELECT $1, status, $2, $3, $4
|
|
FROM users
|
|
WHERE id = $1
|
|
`
|
|
|
|
_, err = s.pool.Exec(ctx, historyQuery, userID, status, reason, changedBy)
|
|
if err != nil {
|
|
// Log error but don't fail the main operation
|
|
fmt.Printf("Failed to log user status change: %v\n", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func containsAny(body string, terms []string) bool {
|
|
// Case insensitive check
|
|
lower := bytes.ToLower([]byte(body))
|
|
for _, term := range terms {
|
|
if bytes.Contains(lower, []byte(term)) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|