feat: implement nuanced violation system with content deletion
- Replace immediate bans with content deletion + account marking - Hard violations: immediate content deletion, account warning/suspension - Soft violations: content hidden pending moderation/appeal - Add content deletion tracking and account status changes - Implement progressive account status (active warning suspended banned) - Track content deletions, warnings, and suspensions in violation history - Update violation thresholds to be more lenient (3 hard = banned, 8 total = banned) - Add content deletion reason and account status change tracking This creates a more nuanced approach where users get multiple chances before being banned, with clear content removal for serious violations.
This commit is contained in:
Patrick Britton
parent
277fc299b2
commit
997d6437be
|
|
@ -0,0 +1,208 @@
|
|||
-- Updated User Appeal System Migration
|
||||
-- Changes: More nuanced violation handling - content deletion + account marking instead of immediate bans
|
||||
|
||||
-- Update user_violations table to include content deletion tracking
|
||||
ALTER TABLE user_violations ADD COLUMN IF NOT EXISTS content_deleted BOOLEAN DEFAULT false;
|
||||
ALTER TABLE user_violations ADD COLUMN IF NOT EXISTS content_deletion_reason TEXT;
|
||||
ALTER TABLE user_violations ADD COLUMN IF NOT EXISTS account_status_change VARCHAR(20) DEFAULT 'none';
|
||||
|
||||
-- Update user_violation_history table
|
||||
ALTER TABLE user_violation_history ADD COLUMN IF NOT EXISTS content_deletions INTEGER DEFAULT 0;
|
||||
ALTER TABLE user_violation_history ADD COLUMN IF NOT EXISTS account_warnings INTEGER DEFAULT 0;
|
||||
ALTER TABLE user_violation_history ADD COLUMN IF NOT EXISTS account_suspensions INTEGER DEFAULT 0;
|
||||
|
||||
-- Update appeal guidelines to reflect new approach
|
||||
UPDATE appeal_guidelines SET
|
||||
auto_ban_threshold = 999, -- Disable auto-ban
|
||||
hard_violation_ban_threshold = 999, -- Disable auto-ban
|
||||
is_active = true
|
||||
WHERE violation_type IN ('hard_violation', 'soft_violation');
|
||||
|
||||
-- Add new status options for user accounts
|
||||
ALTER TABLE users ADD CONSTRAINT IF NOT EXISTS users_status_check
|
||||
CHECK (status IN ('active', 'warning', 'suspended', 'banned', 'under_review'));
|
||||
|
||||
-- Update the ban checking function to use content deletion instead
|
||||
CREATE OR REPLACE FUNCTION check_user_violation_status(p_user_id UUID) RETURNS TEXT AS $$
|
||||
DECLARE
|
||||
v_hard_count INTEGER;
|
||||
v_total_count INTEGER;
|
||||
v_soft_count INTEGER;
|
||||
v_content_deletions INTEGER;
|
||||
v_new_status TEXT := 'active';
|
||||
BEGIN
|
||||
-- Get counts from last 30 days
|
||||
SELECT
|
||||
COUNT(CASE WHEN violation_type = 'hard_violation' THEN 1 END),
|
||||
COUNT(*),
|
||||
COUNT(CASE WHEN violation_type = 'soft_violation' THEN 1 END),
|
||||
COUNT(CASE WHEN content_deleted = true THEN 1 END)
|
||||
INTO v_hard_count, v_total_count, v_soft_count, v_content_deletions
|
||||
FROM user_violations
|
||||
WHERE user_id = p_user_id
|
||||
AND created_at >= NOW() - INTERVAL '30 days';
|
||||
|
||||
-- Determine account status based on violation patterns
|
||||
IF v_hard_count >= 3 THEN
|
||||
v_new_status := 'banned';
|
||||
ELSIF v_total_count >= 8 THEN
|
||||
v_new_status := 'banned';
|
||||
ELSIF v_hard_count >= 2 THEN
|
||||
v_new_status := 'suspended';
|
||||
ELSIF v_total_count >= 5 THEN
|
||||
v_new_status := 'suspended';
|
||||
ELSIF v_hard_count >= 1 OR v_total_count >= 3 THEN
|
||||
v_new_status := 'warning';
|
||||
END IF;
|
||||
|
||||
-- Update user status if it needs to change
|
||||
IF v_new_status != 'active' THEN
|
||||
UPDATE users
|
||||
SET status = v_new_status, updated_at = NOW()
|
||||
WHERE id = p_user_id AND status != v_new_status;
|
||||
|
||||
-- Update violation history
|
||||
UPDATE user_violation_history
|
||||
SET
|
||||
current_status = v_new_status,
|
||||
content_deletions = v_content_deletions,
|
||||
account_warnings = CASE WHEN v_new_status = 'warning' THEN account_warnings + 1 ELSE account_warnings END,
|
||||
account_suspensions = CASE WHEN v_new_status = 'suspended' THEN account_suspensions + 1 ELSE account_suspensions END,
|
||||
updated_at = NOW()
|
||||
WHERE user_id = p_user_id AND violation_date = CURRENT_DATE;
|
||||
END IF;
|
||||
|
||||
RETURN v_new_status;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- Create function to handle content deletion for violations
|
||||
CREATE OR REPLACE FUNCTION handle_violation_content_deletion(
|
||||
p_user_id UUID,
|
||||
p_moderation_flag_id UUID,
|
||||
p_violation_type TEXT
|
||||
) RETURNS BOOLEAN AS $$
|
||||
DECLARE
|
||||
v_post_id UUID;
|
||||
v_comment_id UUID;
|
||||
v_content_deleted BOOLEAN := false;
|
||||
BEGIN
|
||||
-- Get the content ID from the moderation flag
|
||||
SELECT post_id, comment_id
|
||||
INTO v_post_id, v_comment_id
|
||||
FROM moderation_flags
|
||||
WHERE id = p_moderation_flag_id;
|
||||
|
||||
-- Delete or hide the content based on violation type
|
||||
IF p_violation_type = 'hard_violation' THEN
|
||||
-- Hard violations: delete content immediately
|
||||
IF v_post_id IS NOT NULL THEN
|
||||
UPDATE posts SET status = 'deleted', updated_at = NOW() WHERE id = v_post_id;
|
||||
v_content_deleted := true;
|
||||
END IF;
|
||||
|
||||
IF v_comment_id IS NOT NULL THEN
|
||||
UPDATE comments SET status = 'deleted', updated_at = NOW() WHERE id = v_comment_id;
|
||||
v_content_deleted := true;
|
||||
END IF;
|
||||
|
||||
-- Mark the violation record
|
||||
UPDATE user_violations
|
||||
SET content_deleted = true,
|
||||
content_deletion_reason = 'Hard violation - immediate deletion',
|
||||
account_status_change = CASE
|
||||
WHEN (SELECT COUNT(*) FROM user_violations WHERE user_id = p_user_id AND violation_type = 'hard_violation') >= 2 THEN 'suspended'
|
||||
WHEN (SELECT COUNT(*) FROM user_violations WHERE user_id = p_user_id AND violation_type = 'hard_violation') >= 1 THEN 'warning'
|
||||
ELSE 'none'
|
||||
END,
|
||||
updated_at = NOW()
|
||||
WHERE moderation_flag_id = p_moderation_flag_id;
|
||||
|
||||
ELSIF p_violation_type = 'soft_violation' THEN
|
||||
-- Soft violations: hide content pending appeal
|
||||
IF v_post_id IS NOT NULL THEN
|
||||
UPDATE posts SET status = 'pending_moderation', updated_at = NOW() WHERE id = v_post_id;
|
||||
END IF;
|
||||
|
||||
IF v_comment_id IS NOT NULL THEN
|
||||
UPDATE comments SET status = 'pending_moderation', updated_at = NOW() WHERE id = v_comment_id;
|
||||
END IF;
|
||||
|
||||
-- Mark the violation record
|
||||
UPDATE user_violations
|
||||
SET content_deleted = false,
|
||||
content_deletion_reason = 'Soft violation - pending moderation',
|
||||
account_status_change = 'none',
|
||||
updated_at = NOW()
|
||||
WHERE moderation_flag_id = p_moderation_flag_id;
|
||||
END IF;
|
||||
|
||||
-- Check if user status needs updating
|
||||
PERFORM check_user_violation_status(p_user_id);
|
||||
|
||||
RETURN v_content_deleted;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- Update the create_user_violation function to use the new content handling
|
||||
CREATE OR REPLACE FUNCTION create_user_violation(
|
||||
p_user_id UUID,
|
||||
p_moderation_flag_id UUID,
|
||||
p_flag_reason TEXT,
|
||||
p_scores JSONB
|
||||
) RETURNS UUID AS $$
|
||||
DECLARE
|
||||
v_violation_id UUID;
|
||||
v_violation_type TEXT;
|
||||
v_severity DECIMAL;
|
||||
v_is_appealable BOOLEAN;
|
||||
v_appeal_deadline TIMESTAMP WITH TIME ZONE;
|
||||
BEGIN
|
||||
-- Determine violation type based on scores and reason
|
||||
CASE
|
||||
WHEN p_flag_reason IN ('hate') AND (p_scores->>'hate')::DECIMAL > 0.8 THEN
|
||||
BEGIN
|
||||
v_violation_type := 'hard_violation';
|
||||
v_severity := (p_scores->>'hate')::DECIMAL;
|
||||
v_is_appealable := false;
|
||||
v_appeal_deadline := NULL;
|
||||
END;
|
||||
WHEN p_flag_reason IN ('hate', 'violence', 'sexual') AND (p_scores->>'hate')::DECIMAL > 0.6 THEN
|
||||
BEGIN
|
||||
v_violation_type := 'hard_violation';
|
||||
v_severity := GREATEST((p_scores->>'hate')::DECIMAL, (p_scores->>'greed')::DECIMAL, (p_scores->>'delusion')::DECIMAL);
|
||||
v_is_appealable := false;
|
||||
v_appeal_deadline := NULL;
|
||||
END;
|
||||
ELSE
|
||||
BEGIN
|
||||
v_violation_type := 'soft_violation';
|
||||
v_severity := GREATEST((p_scores->>'hate')::DECIMAL, (p_scores->>'greed')::DECIMAL, (p_scores->>'delusion')::DECIMAL);
|
||||
v_is_appealable := true;
|
||||
v_appeal_deadline := NOW() + (SELECT appeal_window_hours FROM appeal_guidelines WHERE violation_type = 'soft_violation' AND is_active = true LIMIT 1) * INTERVAL '1 hour';
|
||||
END;
|
||||
END CASE;
|
||||
|
||||
-- Create the violation record
|
||||
INSERT INTO user_violations (user_id, moderation_flag_id, violation_type, violation_reason, severity_score, is_appealable, appeal_deadline)
|
||||
VALUES (p_user_id, p_moderation_flag_id, v_violation_type, p_flag_reason, v_severity, v_is_appealable, v_appeal_deadline)
|
||||
RETURNING id INTO v_violation_id;
|
||||
|
||||
-- Update violation history
|
||||
INSERT INTO user_violation_history (user_id, violation_date, total_violations, hard_violations, soft_violations)
|
||||
VALUES (p_user_id, CURRENT_DATE, 1,
|
||||
CASE WHEN v_violation_type = 'hard_violation' THEN 1 ELSE 0 END,
|
||||
CASE WHEN v_violation_type = 'soft_violation' THEN 1 ELSE 0 END)
|
||||
ON CONFLICT (user_id, violation_date)
|
||||
DO UPDATE SET
|
||||
total_violations = user_violation_history.total_violations + 1,
|
||||
hard_violations = user_violation_history.hard_violations + CASE WHEN v_violation_type = 'hard_violation' THEN 1 ELSE 0 END,
|
||||
soft_violations = user_violation_history.soft_violations + CASE WHEN v_violation_type = 'soft_violation' THEN 1 ELSE 0 END,
|
||||
updated_at = NOW();
|
||||
|
||||
-- Handle content deletion based on violation type
|
||||
PERFORM handle_violation_content_deletion(p_user_id, p_moderation_flag_id, v_violation_type);
|
||||
|
||||
RETURN v_violation_id;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
|
@ -2,6 +2,7 @@ package models
|
|||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
|
|
@ -22,6 +23,16 @@ const (
|
|||
ViolationStatusExpired ViolationStatus = "expired"
|
||||
)
|
||||
|
||||
type AccountStatus string
|
||||
|
||||
const (
|
||||
AccountStatusActive AccountStatus = "active"
|
||||
AccountStatusWarning AccountStatus = "warning"
|
||||
AccountStatusSuspended AccountStatus = "suspended"
|
||||
AccountStatusBanned AccountStatus = "banned"
|
||||
AccountStatusUnderReview AccountStatus = "under_review"
|
||||
)
|
||||
|
||||
type AppealStatus string
|
||||
|
||||
const (
|
||||
|
|
@ -42,6 +53,9 @@ type UserViolation struct {
|
|||
IsAppealable bool `json:"is_appealable" db:"is_appealable"`
|
||||
AppealDeadline *time.Time `json:"appeal_deadline" db:"appeal_deadline"`
|
||||
Status ViolationStatus `json:"status" db:"status"`
|
||||
ContentDeleted bool `json:"content_deleted" db:"content_deleted"`
|
||||
ContentDeletionReason string `json:"content_deletion_reason" db:"content_deletion_reason"`
|
||||
AccountStatusChange string `json:"account_status_change" db:"account_status_change"`
|
||||
CreatedAt time.Time `json:"created_at" db:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
|
||||
}
|
||||
|
|
@ -71,6 +85,9 @@ type UserViolationHistory struct {
|
|||
AppealsFiled int `json:"appeals_filed" db:"appeals_filed"`
|
||||
AppealsUpheld int `json:"appeals_upheld" db:"appeals_upheld"`
|
||||
AppealsOverturned int `json:"appeals_overturned" db:"appeals_overturned"`
|
||||
ContentDeletions int `json:"content_deletions" db:"content_deletions"`
|
||||
AccountWarnings int `json:"account_warnings" db:"account_warnings"`
|
||||
AccountSuspensions int `json:"account_suspensions" db:"account_suspensions"`
|
||||
CurrentStatus string `json:"current_status" db:"current_status"`
|
||||
BanExpiry *time.Time `json:"ban_expiry" db:"ban_expiry"`
|
||||
CreatedAt time.Time `json:"created_at" db:"created_at"`
|
||||
|
|
|
|||
Loading…
Reference in a new issue