patrick/sojorn
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Fix moderation: use text-moderation-latest model, fix FlagPost/FlagComment queries, add violation functions

Browse source
This commit is contained in:
Patrick Britton 2026-02-06 11:32:41 -06:00
parent a87fcb60b6
commit cc7c39ac33
2 changed files with 88 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
go-backend/internal/services/moderation_service.go
View file

@ -154,7 +154,7 @@ func (s *ModerationService) analyzeWithOpenAI(ctx context.Context, content strin
requestBody := map[string]interface{}{ requestBody := map[string]interface{}{
"input": content, "input": content,
"model": "omni-moderation-latest", "model": "text-moderation-latest",
} }
jsonBody, err := json.Marshal(requestBody) jsonBody, err := json.Marshal(requestBody)
@ -356,32 +356,24 @@ func (s *ModerationService) FlagPost(ctx context.Context, postID uuid.UUID, scor
return fmt.Errorf("failed to marshal scores: %w", err) return fmt.Errorf("failed to marshal scores: %w", err)
} }
query := ` var flagID uuid.UUID
err = s.pool.QueryRow(ctx, `
INSERT INTO moderation_flags (post_id, flag_reason, scores, status) INSERT INTO moderation_flags (post_id, flag_reason, scores, status)
VALUES ($1, $2, $3, 'pending') VALUES ($1, $2, $3, 'pending')
RETURNING id, created_at, user_id RETURNING id
` `, postID, reason, scoresJSON).Scan(&flagID)
var flagID uuid.UUID
var createdAt time.Time
var userID uuid.UUID
err = s.pool.QueryRow(ctx, query, postID, reason, scoresJSON).Scan(&flagID, &createdAt, &userID)
if err != nil { if err != nil {
return fmt.Errorf("failed to insert moderation flag: %w", err) return fmt.Errorf("failed to insert moderation flag: %w", err)
} }
// Create user violation record if we have the user ID fmt.Printf("Moderation flag created: id=%s post=%s reason=%s\n", flagID, postID, reason)
if userID != uuid.Nil {
// This would require the AppealService, but for now we'll create a simple violation record
violationQuery := `
SELECT create_user_violation($1, $2, $3, $4)
`
// Look up the post author and create a violation record
var authorID uuid.UUID
if err := s.pool.QueryRow(ctx, `SELECT author_id FROM posts WHERE id = $1`, postID).Scan(&authorID); err == nil && authorID != uuid.Nil {
var violationID uuid.UUID var violationID uuid.UUID
violationErr := s.pool.QueryRow(ctx, violationQuery, userID, flagID, reason, scoresJSON).Scan(&violationID) if err := s.pool.QueryRow(ctx, `SELECT create_user_violation($1, $2, $3, $4)`, authorID, flagID, reason, scoresJSON).Scan(&violationID); err != nil {
if violationErr != nil { fmt.Printf("Failed to create user violation: %v\n", err)
// Log error but don't fail the flagging process
fmt.Printf("Failed to create user violation: %v\n", violationErr)
} }
} }
@ -394,32 +386,24 @@ func (s *ModerationService) FlagComment(ctx context.Context, commentID uuid.UUID
return fmt.Errorf("failed to marshal scores: %w", err) return fmt.Errorf("failed to marshal scores: %w", err)
} }
query := ` var flagID uuid.UUID
err = s.pool.QueryRow(ctx, `
INSERT INTO moderation_flags (comment_id, flag_reason, scores, status) INSERT INTO moderation_flags (comment_id, flag_reason, scores, status)
VALUES ($1, $2, $3, 'pending') VALUES ($1, $2, $3, 'pending')
RETURNING id, created_at, user_id RETURNING id
` `, commentID, reason, scoresJSON).Scan(&flagID)
var flagID uuid.UUID
var createdAt time.Time
var userID uuid.UUID
err = s.pool.QueryRow(ctx, query, commentID, reason, scoresJSON).Scan(&flagID, &createdAt, &userID)
if err != nil { if err != nil {
return fmt.Errorf("failed to insert comment moderation flag: %w", err) return fmt.Errorf("failed to insert comment moderation flag: %w", err)
} }
// Create user violation record if we have the user ID fmt.Printf("Moderation flag created: id=%s comment=%s reason=%s\n", flagID, commentID, reason)
if userID != uuid.Nil {
// This would require the AppealService, but for now we'll create a simple violation record
violationQuery := `
SELECT create_user_violation($1, $2, $3, $4)
`
// Look up the comment author and create a violation record
var authorID uuid.UUID
if err := s.pool.QueryRow(ctx, `SELECT author_id FROM comments WHERE id = $1`, commentID).Scan(&authorID); err == nil && authorID != uuid.Nil {
var violationID uuid.UUID var violationID uuid.UUID
violationErr := s.pool.QueryRow(ctx, violationQuery, userID, flagID, reason, scoresJSON).Scan(&violationID) if err := s.pool.QueryRow(ctx, `SELECT create_user_violation($1, $2, $3, $4)`, authorID, flagID, reason, scoresJSON).Scan(&violationID); err != nil {
if violationErr != nil { fmt.Printf("Failed to create user violation: %v\n", err)
// Log error but don't fail the flagging process
fmt.Printf("Failed to create user violation: %v\n", violationErr)
} }
} }

67
go-backend/scripts/create_violation_functions.sql Normal file
View file

@ -0,0 +1,67 @@
CREATE OR REPLACE FUNCTION create_user_violation(
p_user_id UUID,
p_moderation_flag_id UUID,
p_flag_reason TEXT,
p_scores JSONB
) RETURNS UUID AS $$
DECLARE
v_violation_id UUID;
v_violation_type TEXT;
v_severity DECIMAL;
v_is_appealable BOOLEAN;
v_appeal_deadline TIMESTAMP WITH TIME ZONE;
BEGIN
CASE
WHEN p_flag_reason IN ('hate') AND (p_scores->>'hate')::DECIMAL > 0.8 THEN
v_violation_type := 'hard_violation';
v_severity := (p_scores->>'hate')::DECIMAL;
v_is_appealable := false;
v_appeal_deadline := NULL;
WHEN p_flag_reason IN ('hate', 'violence', 'sexual') AND (p_scores->>'hate')::DECIMAL > 0.6 THEN
v_violation_type := 'hard_violation';
v_severity := GREATEST((p_scores->>'hate')::DECIMAL, (p_scores->>'greed')::DECIMAL, (p_scores->>'delusion')::DECIMAL);
v_is_appealable := false;
v_appeal_deadline := NULL;
ELSE
v_violation_type := 'soft_violation';
v_severity := GREATEST((p_scores->>'hate')::DECIMAL, (p_scores->>'greed')::DECIMAL, (p_scores->>'delusion')::DECIMAL);
v_is_appealable := true;
v_appeal_deadline := NOW() + INTERVAL '72 hours';
END CASE;
INSERT INTO user_violations (user_id, moderation_flag_id, violation_type, violation_reason, severity_score, is_appealable, appeal_deadline)
VALUES (p_user_id, p_moderation_flag_id, v_violation_type, p_flag_reason, v_severity, v_is_appealable, v_appeal_deadline)
RETURNING id INTO v_violation_id;
INSERT INTO user_violation_history (user_id, violation_date, total_violations, hard_violations, soft_violations)
VALUES (p_user_id, CURRENT_DATE, 1,
CASE WHEN v_violation_type = 'hard_violation' THEN 1 ELSE 0 END,
CASE WHEN v_violation_type = 'soft_violation' THEN 1 ELSE 0 END)
ON CONFLICT (user_id, violation_date)
DO UPDATE SET
total_violations = user_violation_history.total_violations + 1,
hard_violations = user_violation_history.hard_violations + CASE WHEN v_violation_type = 'hard_violation' THEN 1 ELSE 0 END,
soft_violations = user_violation_history.soft_violations + CASE WHEN v_violation_type = 'soft_violation' THEN 1 ELSE 0 END,
updated_at = NOW();
RETURN v_violation_id;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION check_user_ban_status(p_user_id UUID) RETURNS BOOLEAN AS $$
DECLARE
v_hard_count INTEGER;
v_total_count INTEGER;
BEGIN
SELECT COUNT(*), COALESCE(SUM(CASE WHEN violation_type = 'hard_violation' THEN 1 ELSE 0 END), 0)
INTO v_total_count, v_hard_count
FROM user_violations
WHERE user_id = p_user_id AND created_at >= NOW() - INTERVAL '30 days';
IF v_hard_count >= 2 OR v_total_count >= 5 THEN
UPDATE users SET status = 'banned', updated_at = NOW() WHERE id = p_user_id;
RETURN true;
END IF;
RETURN false;
END;
$$ LANGUAGE plpgsql;