Fix moderation: use text-moderation-latest model, fix FlagPost/FlagComment queries, add violation functions
This commit is contained in:
Patrick Britton
parent
a87fcb60b6
commit
cc7c39ac33
|
|
@ -154,7 +154,7 @@ func (s *ModerationService) analyzeWithOpenAI(ctx context.Context, content strin
|
|||
|
||||
requestBody := map[string]interface{}{
|
||||
"input": content,
|
||||
"model": "omni-moderation-latest",
|
||||
"model": "text-moderation-latest",
|
||||
}
|
||||
|
||||
jsonBody, err := json.Marshal(requestBody)
|
||||
|
|
@ -356,32 +356,24 @@ func (s *ModerationService) FlagPost(ctx context.Context, postID uuid.UUID, scor
|
|||
return fmt.Errorf("failed to marshal scores: %w", err)
|
||||
}
|
||||
|
||||
query := `
|
||||
var flagID uuid.UUID
|
||||
err = s.pool.QueryRow(ctx, `
|
||||
INSERT INTO moderation_flags (post_id, flag_reason, scores, status)
|
||||
VALUES ($1, $2, $3, 'pending')
|
||||
RETURNING id, created_at, user_id
|
||||
`
|
||||
|
||||
var flagID uuid.UUID
|
||||
var createdAt time.Time
|
||||
var userID uuid.UUID
|
||||
err = s.pool.QueryRow(ctx, query, postID, reason, scoresJSON).Scan(&flagID, &createdAt, &userID)
|
||||
RETURNING id
|
||||
`, postID, reason, scoresJSON).Scan(&flagID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to insert moderation flag: %w", err)
|
||||
}
|
||||
|
||||
// Create user violation record if we have the user ID
|
||||
if userID != uuid.Nil {
|
||||
// This would require the AppealService, but for now we'll create a simple violation record
|
||||
violationQuery := `
|
||||
SELECT create_user_violation($1, $2, $3, $4)
|
||||
`
|
||||
fmt.Printf("Moderation flag created: id=%s post=%s reason=%s\n", flagID, postID, reason)
|
||||
|
||||
// Look up the post author and create a violation record
|
||||
var authorID uuid.UUID
|
||||
if err := s.pool.QueryRow(ctx, `SELECT author_id FROM posts WHERE id = $1`, postID).Scan(&authorID); err == nil && authorID != uuid.Nil {
|
||||
var violationID uuid.UUID
|
||||
violationErr := s.pool.QueryRow(ctx, violationQuery, userID, flagID, reason, scoresJSON).Scan(&violationID)
|
||||
if violationErr != nil {
|
||||
// Log error but don't fail the flagging process
|
||||
fmt.Printf("Failed to create user violation: %v\n", violationErr)
|
||||
if err := s.pool.QueryRow(ctx, `SELECT create_user_violation($1, $2, $3, $4)`, authorID, flagID, reason, scoresJSON).Scan(&violationID); err != nil {
|
||||
fmt.Printf("Failed to create user violation: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -394,32 +386,24 @@ func (s *ModerationService) FlagComment(ctx context.Context, commentID uuid.UUID
|
|||
return fmt.Errorf("failed to marshal scores: %w", err)
|
||||
}
|
||||
|
||||
query := `
|
||||
var flagID uuid.UUID
|
||||
err = s.pool.QueryRow(ctx, `
|
||||
INSERT INTO moderation_flags (comment_id, flag_reason, scores, status)
|
||||
VALUES ($1, $2, $3, 'pending')
|
||||
RETURNING id, created_at, user_id
|
||||
`
|
||||
|
||||
var flagID uuid.UUID
|
||||
var createdAt time.Time
|
||||
var userID uuid.UUID
|
||||
err = s.pool.QueryRow(ctx, query, commentID, reason, scoresJSON).Scan(&flagID, &createdAt, &userID)
|
||||
RETURNING id
|
||||
`, commentID, reason, scoresJSON).Scan(&flagID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to insert comment moderation flag: %w", err)
|
||||
}
|
||||
|
||||
// Create user violation record if we have the user ID
|
||||
if userID != uuid.Nil {
|
||||
// This would require the AppealService, but for now we'll create a simple violation record
|
||||
violationQuery := `
|
||||
SELECT create_user_violation($1, $2, $3, $4)
|
||||
`
|
||||
fmt.Printf("Moderation flag created: id=%s comment=%s reason=%s\n", flagID, commentID, reason)
|
||||
|
||||
// Look up the comment author and create a violation record
|
||||
var authorID uuid.UUID
|
||||
if err := s.pool.QueryRow(ctx, `SELECT author_id FROM comments WHERE id = $1`, commentID).Scan(&authorID); err == nil && authorID != uuid.Nil {
|
||||
var violationID uuid.UUID
|
||||
violationErr := s.pool.QueryRow(ctx, violationQuery, userID, flagID, reason, scoresJSON).Scan(&violationID)
|
||||
if violationErr != nil {
|
||||
// Log error but don't fail the flagging process
|
||||
fmt.Printf("Failed to create user violation: %v\n", violationErr)
|
||||
if err := s.pool.QueryRow(ctx, `SELECT create_user_violation($1, $2, $3, $4)`, authorID, flagID, reason, scoresJSON).Scan(&violationID); err != nil {
|
||||
fmt.Printf("Failed to create user violation: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
67
go-backend/scripts/create_violation_functions.sql
Normal file
67
go-backend/scripts/create_violation_functions.sql
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
CREATE OR REPLACE FUNCTION create_user_violation(
|
||||
p_user_id UUID,
|
||||
p_moderation_flag_id UUID,
|
||||
p_flag_reason TEXT,
|
||||
p_scores JSONB
|
||||
) RETURNS UUID AS $$
|
||||
DECLARE
|
||||
v_violation_id UUID;
|
||||
v_violation_type TEXT;
|
||||
v_severity DECIMAL;
|
||||
v_is_appealable BOOLEAN;
|
||||
v_appeal_deadline TIMESTAMP WITH TIME ZONE;
|
||||
BEGIN
|
||||
CASE
|
||||
WHEN p_flag_reason IN ('hate') AND (p_scores->>'hate')::DECIMAL > 0.8 THEN
|
||||
v_violation_type := 'hard_violation';
|
||||
v_severity := (p_scores->>'hate')::DECIMAL;
|
||||
v_is_appealable := false;
|
||||
v_appeal_deadline := NULL;
|
||||
WHEN p_flag_reason IN ('hate', 'violence', 'sexual') AND (p_scores->>'hate')::DECIMAL > 0.6 THEN
|
||||
v_violation_type := 'hard_violation';
|
||||
v_severity := GREATEST((p_scores->>'hate')::DECIMAL, (p_scores->>'greed')::DECIMAL, (p_scores->>'delusion')::DECIMAL);
|
||||
v_is_appealable := false;
|
||||
v_appeal_deadline := NULL;
|
||||
ELSE
|
||||
v_violation_type := 'soft_violation';
|
||||
v_severity := GREATEST((p_scores->>'hate')::DECIMAL, (p_scores->>'greed')::DECIMAL, (p_scores->>'delusion')::DECIMAL);
|
||||
v_is_appealable := true;
|
||||
v_appeal_deadline := NOW() + INTERVAL '72 hours';
|
||||
END CASE;
|
||||
|
||||
INSERT INTO user_violations (user_id, moderation_flag_id, violation_type, violation_reason, severity_score, is_appealable, appeal_deadline)
|
||||
VALUES (p_user_id, p_moderation_flag_id, v_violation_type, p_flag_reason, v_severity, v_is_appealable, v_appeal_deadline)
|
||||
RETURNING id INTO v_violation_id;
|
||||
|
||||
INSERT INTO user_violation_history (user_id, violation_date, total_violations, hard_violations, soft_violations)
|
||||
VALUES (p_user_id, CURRENT_DATE, 1,
|
||||
CASE WHEN v_violation_type = 'hard_violation' THEN 1 ELSE 0 END,
|
||||
CASE WHEN v_violation_type = 'soft_violation' THEN 1 ELSE 0 END)
|
||||
ON CONFLICT (user_id, violation_date)
|
||||
DO UPDATE SET
|
||||
total_violations = user_violation_history.total_violations + 1,
|
||||
hard_violations = user_violation_history.hard_violations + CASE WHEN v_violation_type = 'hard_violation' THEN 1 ELSE 0 END,
|
||||
soft_violations = user_violation_history.soft_violations + CASE WHEN v_violation_type = 'soft_violation' THEN 1 ELSE 0 END,
|
||||
updated_at = NOW();
|
||||
|
||||
RETURN v_violation_id;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE OR REPLACE FUNCTION check_user_ban_status(p_user_id UUID) RETURNS BOOLEAN AS $$
|
||||
DECLARE
|
||||
v_hard_count INTEGER;
|
||||
v_total_count INTEGER;
|
||||
BEGIN
|
||||
SELECT COUNT(*), COALESCE(SUM(CASE WHEN violation_type = 'hard_violation' THEN 1 ELSE 0 END), 0)
|
||||
INTO v_total_count, v_hard_count
|
||||
FROM user_violations
|
||||
WHERE user_id = p_user_id AND created_at >= NOW() - INTERVAL '30 days';
|
||||
|
||||
IF v_hard_count >= 2 OR v_total_count >= 5 THEN
|
||||
UPDATE users SET status = 'banned', updated_at = NOW() WHERE id = p_user_id;
|
||||
RETURN true;
|
||||
END IF;
|
||||
RETURN false;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
Loading…
Reference in a new issue