diff --git a/go-backend/internal/services/moderation_service.go b/go-backend/internal/services/moderation_service.go index 5016251..9170dfb 100644 --- a/go-backend/internal/services/moderation_service.go +++ b/go-backend/internal/services/moderation_service.go @@ -154,7 +154,7 @@ func (s *ModerationService) analyzeWithOpenAI(ctx context.Context, content strin requestBody := map[string]interface{}{ "input": content, - "model": "omni-moderation-latest", + "model": "text-moderation-latest", } jsonBody, err := json.Marshal(requestBody) @@ -356,32 +356,24 @@ func (s *ModerationService) FlagPost(ctx context.Context, postID uuid.UUID, scor return fmt.Errorf("failed to marshal scores: %w", err) } - query := ` + var flagID uuid.UUID + err = s.pool.QueryRow(ctx, ` INSERT INTO moderation_flags (post_id, flag_reason, scores, status) VALUES ($1, $2, $3, 'pending') - RETURNING id, created_at, user_id - ` - - var flagID uuid.UUID - var createdAt time.Time - var userID uuid.UUID - err = s.pool.QueryRow(ctx, query, postID, reason, scoresJSON).Scan(&flagID, &createdAt, &userID) + RETURNING id + `, postID, reason, scoresJSON).Scan(&flagID) if err != nil { return fmt.Errorf("failed to insert moderation flag: %w", err) } - // Create user violation record if we have the user ID - if userID != uuid.Nil { - // This would require the AppealService, but for now we'll create a simple violation record - violationQuery := ` - SELECT create_user_violation($1, $2, $3, $4) - ` + fmt.Printf("Moderation flag created: id=%s post=%s reason=%s\n", flagID, postID, reason) + // Look up the post author and create a violation record + var authorID uuid.UUID + if err := s.pool.QueryRow(ctx, `SELECT author_id FROM posts WHERE id = $1`, postID).Scan(&authorID); err == nil && authorID != uuid.Nil { var violationID uuid.UUID - violationErr := s.pool.QueryRow(ctx, violationQuery, userID, flagID, reason, scoresJSON).Scan(&violationID) - if violationErr != nil { - // Log error but don't fail the flagging process - fmt.Printf("Failed to create user violation: %v\n", violationErr) + if err := s.pool.QueryRow(ctx, `SELECT create_user_violation($1, $2, $3, $4)`, authorID, flagID, reason, scoresJSON).Scan(&violationID); err != nil { + fmt.Printf("Failed to create user violation: %v\n", err) } } @@ -394,32 +386,24 @@ func (s *ModerationService) FlagComment(ctx context.Context, commentID uuid.UUID return fmt.Errorf("failed to marshal scores: %w", err) } - query := ` + var flagID uuid.UUID + err = s.pool.QueryRow(ctx, ` INSERT INTO moderation_flags (comment_id, flag_reason, scores, status) VALUES ($1, $2, $3, 'pending') - RETURNING id, created_at, user_id - ` - - var flagID uuid.UUID - var createdAt time.Time - var userID uuid.UUID - err = s.pool.QueryRow(ctx, query, commentID, reason, scoresJSON).Scan(&flagID, &createdAt, &userID) + RETURNING id + `, commentID, reason, scoresJSON).Scan(&flagID) if err != nil { return fmt.Errorf("failed to insert comment moderation flag: %w", err) } - // Create user violation record if we have the user ID - if userID != uuid.Nil { - // This would require the AppealService, but for now we'll create a simple violation record - violationQuery := ` - SELECT create_user_violation($1, $2, $3, $4) - ` + fmt.Printf("Moderation flag created: id=%s comment=%s reason=%s\n", flagID, commentID, reason) + // Look up the comment author and create a violation record + var authorID uuid.UUID + if err := s.pool.QueryRow(ctx, `SELECT author_id FROM comments WHERE id = $1`, commentID).Scan(&authorID); err == nil && authorID != uuid.Nil { var violationID uuid.UUID - violationErr := s.pool.QueryRow(ctx, violationQuery, userID, flagID, reason, scoresJSON).Scan(&violationID) - if violationErr != nil { - // Log error but don't fail the flagging process - fmt.Printf("Failed to create user violation: %v\n", violationErr) + if err := s.pool.QueryRow(ctx, `SELECT create_user_violation($1, $2, $3, $4)`, authorID, flagID, reason, scoresJSON).Scan(&violationID); err != nil { + fmt.Printf("Failed to create user violation: %v\n", err) } } diff --git a/go-backend/scripts/create_violation_functions.sql b/go-backend/scripts/create_violation_functions.sql new file mode 100644 index 0000000..d8af272 --- /dev/null +++ b/go-backend/scripts/create_violation_functions.sql @@ -0,0 +1,67 @@ +CREATE OR REPLACE FUNCTION create_user_violation( + p_user_id UUID, + p_moderation_flag_id UUID, + p_flag_reason TEXT, + p_scores JSONB +) RETURNS UUID AS $$ +DECLARE + v_violation_id UUID; + v_violation_type TEXT; + v_severity DECIMAL; + v_is_appealable BOOLEAN; + v_appeal_deadline TIMESTAMP WITH TIME ZONE; +BEGIN + CASE + WHEN p_flag_reason IN ('hate') AND (p_scores->>'hate')::DECIMAL > 0.8 THEN + v_violation_type := 'hard_violation'; + v_severity := (p_scores->>'hate')::DECIMAL; + v_is_appealable := false; + v_appeal_deadline := NULL; + WHEN p_flag_reason IN ('hate', 'violence', 'sexual') AND (p_scores->>'hate')::DECIMAL > 0.6 THEN + v_violation_type := 'hard_violation'; + v_severity := GREATEST((p_scores->>'hate')::DECIMAL, (p_scores->>'greed')::DECIMAL, (p_scores->>'delusion')::DECIMAL); + v_is_appealable := false; + v_appeal_deadline := NULL; + ELSE + v_violation_type := 'soft_violation'; + v_severity := GREATEST((p_scores->>'hate')::DECIMAL, (p_scores->>'greed')::DECIMAL, (p_scores->>'delusion')::DECIMAL); + v_is_appealable := true; + v_appeal_deadline := NOW() + INTERVAL '72 hours'; + END CASE; + + INSERT INTO user_violations (user_id, moderation_flag_id, violation_type, violation_reason, severity_score, is_appealable, appeal_deadline) + VALUES (p_user_id, p_moderation_flag_id, v_violation_type, p_flag_reason, v_severity, v_is_appealable, v_appeal_deadline) + RETURNING id INTO v_violation_id; + + INSERT INTO user_violation_history (user_id, violation_date, total_violations, hard_violations, soft_violations) + VALUES (p_user_id, CURRENT_DATE, 1, + CASE WHEN v_violation_type = 'hard_violation' THEN 1 ELSE 0 END, + CASE WHEN v_violation_type = 'soft_violation' THEN 1 ELSE 0 END) + ON CONFLICT (user_id, violation_date) + DO UPDATE SET + total_violations = user_violation_history.total_violations + 1, + hard_violations = user_violation_history.hard_violations + CASE WHEN v_violation_type = 'hard_violation' THEN 1 ELSE 0 END, + soft_violations = user_violation_history.soft_violations + CASE WHEN v_violation_type = 'soft_violation' THEN 1 ELSE 0 END, + updated_at = NOW(); + + RETURN v_violation_id; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION check_user_ban_status(p_user_id UUID) RETURNS BOOLEAN AS $$ +DECLARE + v_hard_count INTEGER; + v_total_count INTEGER; +BEGIN + SELECT COUNT(*), COALESCE(SUM(CASE WHEN violation_type = 'hard_violation' THEN 1 ELSE 0 END), 0) + INTO v_total_count, v_hard_count + FROM user_violations + WHERE user_id = p_user_id AND created_at >= NOW() - INTERVAL '30 days'; + + IF v_hard_count >= 2 OR v_total_count >= 5 THEN + UPDATE users SET status = 'banned', updated_at = NOW() WHERE id = p_user_id; + RETURN true; + END IF; + RETURN false; +END; +$$ LANGUAGE plpgsql;