feat: implement nuanced violation system with content deletion

- Replace immediate bans with content deletion + account marking
- Hard violations: immediate content deletion, account warning/suspension
- Soft violations: content hidden pending moderation/appeal
- Add content deletion tracking and account status changes
- Implement progressive account status (active  warning  suspended  banned)
- Track content deletions, warnings, and suspensions in violation history
- Update violation thresholds to be more lenient (3 hard = banned, 8 total = banned)
- Add content deletion reason and account status change tracking

This creates a more nuanced approach where users get multiple chances
before being banned, with clear content removal for serious violations.
This commit is contained in:
Patrick Britton 2026-02-05 07:59:35 -06:00
parent 277fc299b2
commit 997d6437be
2 changed files with 279 additions and 54 deletions

View file

@ -0,0 +1,208 @@
-- Updated User Appeal System Migration
-- Changes: More nuanced violation handling - content deletion + account marking instead of immediate bans
-- Update user_violations table to include content deletion tracking
ALTER TABLE user_violations ADD COLUMN IF NOT EXISTS content_deleted BOOLEAN DEFAULT false;
ALTER TABLE user_violations ADD COLUMN IF NOT EXISTS content_deletion_reason TEXT;
ALTER TABLE user_violations ADD COLUMN IF NOT EXISTS account_status_change VARCHAR(20) DEFAULT 'none';
-- Update user_violation_history table
ALTER TABLE user_violation_history ADD COLUMN IF NOT EXISTS content_deletions INTEGER DEFAULT 0;
ALTER TABLE user_violation_history ADD COLUMN IF NOT EXISTS account_warnings INTEGER DEFAULT 0;
ALTER TABLE user_violation_history ADD COLUMN IF NOT EXISTS account_suspensions INTEGER DEFAULT 0;
-- Update appeal guidelines to reflect new approach
UPDATE appeal_guidelines SET
auto_ban_threshold = 999, -- Disable auto-ban
hard_violation_ban_threshold = 999, -- Disable auto-ban
is_active = true
WHERE violation_type IN ('hard_violation', 'soft_violation');
-- Add new status options for user accounts
ALTER TABLE users ADD CONSTRAINT IF NOT EXISTS users_status_check
CHECK (status IN ('active', 'warning', 'suspended', 'banned', 'under_review'));
-- Update the ban checking function to use content deletion instead
CREATE OR REPLACE FUNCTION check_user_violation_status(p_user_id UUID) RETURNS TEXT AS $$
DECLARE
v_hard_count INTEGER;
v_total_count INTEGER;
v_soft_count INTEGER;
v_content_deletions INTEGER;
v_new_status TEXT := 'active';
BEGIN
-- Get counts from last 30 days
SELECT
COUNT(CASE WHEN violation_type = 'hard_violation' THEN 1 END),
COUNT(*),
COUNT(CASE WHEN violation_type = 'soft_violation' THEN 1 END),
COUNT(CASE WHEN content_deleted = true THEN 1 END)
INTO v_hard_count, v_total_count, v_soft_count, v_content_deletions
FROM user_violations
WHERE user_id = p_user_id
AND created_at >= NOW() - INTERVAL '30 days';
-- Determine account status based on violation patterns
IF v_hard_count >= 3 THEN
v_new_status := 'banned';
ELSIF v_total_count >= 8 THEN
v_new_status := 'banned';
ELSIF v_hard_count >= 2 THEN
v_new_status := 'suspended';
ELSIF v_total_count >= 5 THEN
v_new_status := 'suspended';
ELSIF v_hard_count >= 1 OR v_total_count >= 3 THEN
v_new_status := 'warning';
END IF;
-- Update user status if it needs to change
IF v_new_status != 'active' THEN
UPDATE users
SET status = v_new_status, updated_at = NOW()
WHERE id = p_user_id AND status != v_new_status;
-- Update violation history
UPDATE user_violation_history
SET
current_status = v_new_status,
content_deletions = v_content_deletions,
account_warnings = CASE WHEN v_new_status = 'warning' THEN account_warnings + 1 ELSE account_warnings END,
account_suspensions = CASE WHEN v_new_status = 'suspended' THEN account_suspensions + 1 ELSE account_suspensions END,
updated_at = NOW()
WHERE user_id = p_user_id AND violation_date = CURRENT_DATE;
END IF;
RETURN v_new_status;
END;
$$ LANGUAGE plpgsql;
-- Create function to handle content deletion for violations
CREATE OR REPLACE FUNCTION handle_violation_content_deletion(
p_user_id UUID,
p_moderation_flag_id UUID,
p_violation_type TEXT
) RETURNS BOOLEAN AS $$
DECLARE
v_post_id UUID;
v_comment_id UUID;
v_content_deleted BOOLEAN := false;
BEGIN
-- Get the content ID from the moderation flag
SELECT post_id, comment_id
INTO v_post_id, v_comment_id
FROM moderation_flags
WHERE id = p_moderation_flag_id;
-- Delete or hide the content based on violation type
IF p_violation_type = 'hard_violation' THEN
-- Hard violations: delete content immediately
IF v_post_id IS NOT NULL THEN
UPDATE posts SET status = 'deleted', updated_at = NOW() WHERE id = v_post_id;
v_content_deleted := true;
END IF;
IF v_comment_id IS NOT NULL THEN
UPDATE comments SET status = 'deleted', updated_at = NOW() WHERE id = v_comment_id;
v_content_deleted := true;
END IF;
-- Mark the violation record
UPDATE user_violations
SET content_deleted = true,
content_deletion_reason = 'Hard violation - immediate deletion',
account_status_change = CASE
WHEN (SELECT COUNT(*) FROM user_violations WHERE user_id = p_user_id AND violation_type = 'hard_violation') >= 2 THEN 'suspended'
WHEN (SELECT COUNT(*) FROM user_violations WHERE user_id = p_user_id AND violation_type = 'hard_violation') >= 1 THEN 'warning'
ELSE 'none'
END,
updated_at = NOW()
WHERE moderation_flag_id = p_moderation_flag_id;
ELSIF p_violation_type = 'soft_violation' THEN
-- Soft violations: hide content pending appeal
IF v_post_id IS NOT NULL THEN
UPDATE posts SET status = 'pending_moderation', updated_at = NOW() WHERE id = v_post_id;
END IF;
IF v_comment_id IS NOT NULL THEN
UPDATE comments SET status = 'pending_moderation', updated_at = NOW() WHERE id = v_comment_id;
END IF;
-- Mark the violation record
UPDATE user_violations
SET content_deleted = false,
content_deletion_reason = 'Soft violation - pending moderation',
account_status_change = 'none',
updated_at = NOW()
WHERE moderation_flag_id = p_moderation_flag_id;
END IF;
-- Check if user status needs updating
PERFORM check_user_violation_status(p_user_id);
RETURN v_content_deleted;
END;
$$ LANGUAGE plpgsql;
-- Update the create_user_violation function to use the new content handling
CREATE OR REPLACE FUNCTION create_user_violation(
p_user_id UUID,
p_moderation_flag_id UUID,
p_flag_reason TEXT,
p_scores JSONB
) RETURNS UUID AS $$
DECLARE
v_violation_id UUID;
v_violation_type TEXT;
v_severity DECIMAL;
v_is_appealable BOOLEAN;
v_appeal_deadline TIMESTAMP WITH TIME ZONE;
BEGIN
-- Determine violation type based on scores and reason
CASE
WHEN p_flag_reason IN ('hate') AND (p_scores->>'hate')::DECIMAL > 0.8 THEN
BEGIN
v_violation_type := 'hard_violation';
v_severity := (p_scores->>'hate')::DECIMAL;
v_is_appealable := false;
v_appeal_deadline := NULL;
END;
WHEN p_flag_reason IN ('hate', 'violence', 'sexual') AND (p_scores->>'hate')::DECIMAL > 0.6 THEN
BEGIN
v_violation_type := 'hard_violation';
v_severity := GREATEST((p_scores->>'hate')::DECIMAL, (p_scores->>'greed')::DECIMAL, (p_scores->>'delusion')::DECIMAL);
v_is_appealable := false;
v_appeal_deadline := NULL;
END;
ELSE
BEGIN
v_violation_type := 'soft_violation';
v_severity := GREATEST((p_scores->>'hate')::DECIMAL, (p_scores->>'greed')::DECIMAL, (p_scores->>'delusion')::DECIMAL);
v_is_appealable := true;
v_appeal_deadline := NOW() + (SELECT appeal_window_hours FROM appeal_guidelines WHERE violation_type = 'soft_violation' AND is_active = true LIMIT 1) * INTERVAL '1 hour';
END;
END CASE;
-- Create the violation record
INSERT INTO user_violations (user_id, moderation_flag_id, violation_type, violation_reason, severity_score, is_appealable, appeal_deadline)
VALUES (p_user_id, p_moderation_flag_id, v_violation_type, p_flag_reason, v_severity, v_is_appealable, v_appeal_deadline)
RETURNING id INTO v_violation_id;
-- Update violation history
INSERT INTO user_violation_history (user_id, violation_date, total_violations, hard_violations, soft_violations)
VALUES (p_user_id, CURRENT_DATE, 1,
CASE WHEN v_violation_type = 'hard_violation' THEN 1 ELSE 0 END,
CASE WHEN v_violation_type = 'soft_violation' THEN 1 ELSE 0 END)
ON CONFLICT (user_id, violation_date)
DO UPDATE SET
total_violations = user_violation_history.total_violations + 1,
hard_violations = user_violation_history.hard_violations + CASE WHEN v_violation_type = 'hard_violation' THEN 1 ELSE 0 END,
soft_violations = user_violation_history.soft_violations + CASE WHEN v_violation_type = 'soft_violation' THEN 1 ELSE 0 END,
updated_at = NOW();
-- Handle content deletion based on violation type
PERFORM handle_violation_content_deletion(p_user_id, p_moderation_flag_id, v_violation_type);
RETURN v_violation_id;
END;
$$ LANGUAGE plpgsql;

View file

@ -2,24 +2,35 @@ package models
import ( import (
"time" "time"
"github.com/google/uuid" "github.com/google/uuid"
) )
type ViolationType string type ViolationType string
const ( const (
ViolationTypeHard ViolationType = "hard_violation" ViolationTypeHard ViolationType = "hard_violation"
ViolationTypeSoft ViolationType = "soft_violation" ViolationTypeSoft ViolationType = "soft_violation"
) )
type ViolationStatus string type ViolationStatus string
const ( const (
ViolationStatusActive ViolationStatus = "active" ViolationStatusActive ViolationStatus = "active"
ViolationStatusAppealed ViolationStatus = "appealed" ViolationStatusAppealed ViolationStatus = "appealed"
ViolationStatusUpheld ViolationStatus = "upheld" ViolationStatusUpheld ViolationStatus = "upheld"
ViolationStatusOverturned ViolationStatus = "overturned" ViolationStatusOverturned ViolationStatus = "overturned"
ViolationStatusExpired ViolationStatus = "expired" ViolationStatusExpired ViolationStatus = "expired"
)
type AccountStatus string
const (
AccountStatusActive AccountStatus = "active"
AccountStatusWarning AccountStatus = "warning"
AccountStatusSuspended AccountStatus = "suspended"
AccountStatusBanned AccountStatus = "banned"
AccountStatusUnderReview AccountStatus = "under_review"
) )
type AppealStatus string type AppealStatus string
@ -33,17 +44,20 @@ const (
) )
type UserViolation struct { type UserViolation struct {
ID uuid.UUID `json:"id" db:"id"` ID uuid.UUID `json:"id" db:"id"`
UserID uuid.UUID `json:"user_id" db:"user_id"` UserID uuid.UUID `json:"user_id" db:"user_id"`
ModerationFlagID uuid.UUID `json:"moderation_flag_id" db:"moderation_flag_id"` ModerationFlagID uuid.UUID `json:"moderation_flag_id" db:"moderation_flag_id"`
ViolationType ViolationType `json:"violation_type" db:"violation_type"` ViolationType ViolationType `json:"violation_type" db:"violation_type"`
ViolationReason string `json:"violation_reason" db:"violation_reason"` ViolationReason string `json:"violation_reason" db:"violation_reason"`
SeverityScore float64 `json:"severity_score" db:"severity_score"` SeverityScore float64 `json:"severity_score" db:"severity_score"`
IsAppealable bool `json:"is_appealable" db:"is_appealable"` IsAppealable bool `json:"is_appealable" db:"is_appealable"`
AppealDeadline *time.Time `json:"appeal_deadline" db:"appeal_deadline"` AppealDeadline *time.Time `json:"appeal_deadline" db:"appeal_deadline"`
Status ViolationStatus `json:"status" db:"status"` Status ViolationStatus `json:"status" db:"status"`
CreatedAt time.Time `json:"created_at" db:"created_at"` ContentDeleted bool `json:"content_deleted" db:"content_deleted"`
UpdatedAt time.Time `json:"updated_at" db:"updated_at"` ContentDeletionReason string `json:"content_deletion_reason" db:"content_deletion_reason"`
AccountStatusChange string `json:"account_status_change" db:"account_status_change"`
CreatedAt time.Time `json:"created_at" db:"created_at"`
UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
} }
type UserAppeal struct { type UserAppeal struct {
@ -62,49 +76,52 @@ type UserAppeal struct {
} }
type UserViolationHistory struct { type UserViolationHistory struct {
ID uuid.UUID `json:"id" db:"id"` ID uuid.UUID `json:"id" db:"id"`
UserID uuid.UUID `json:"user_id" db:"user_id"` UserID uuid.UUID `json:"user_id" db:"user_id"`
ViolationDate time.Time `json:"violation_date" db:"violation_date"` ViolationDate time.Time `json:"violation_date" db:"violation_date"`
TotalViolations int `json:"total_violations" db:"total_violations"` TotalViolations int `json:"total_violations" db:"total_violations"`
HardViolations int `json:"hard_violations" db:"hard_violations"` HardViolations int `json:"hard_violations" db:"hard_violations"`
SoftViolations int `json:"soft_violations" db:"soft_violations"` SoftViolations int `json:"soft_violations" db:"soft_violations"`
AppealsFiled int `json:"appeals_filed" db:"appeals_filed"` AppealsFiled int `json:"appeals_filed" db:"appeals_filed"`
AppealsUpheld int `json:"appeals_upheld" db:"appeals_upheld"` AppealsUpheld int `json:"appeals_upheld" db:"appeals_upheld"`
AppealsOverturned int `json:"appeals_overturned" db:"appeals_overturned"` AppealsOverturned int `json:"appeals_overturned" db:"appeals_overturned"`
CurrentStatus string `json:"current_status" db:"current_status"` ContentDeletions int `json:"content_deletions" db:"content_deletions"`
BanExpiry *time.Time `json:"ban_expiry" db:"ban_expiry"` AccountWarnings int `json:"account_warnings" db:"account_warnings"`
CreatedAt time.Time `json:"created_at" db:"created_at"` AccountSuspensions int `json:"account_suspensions" db:"account_suspensions"`
UpdatedAt time.Time `json:"updated_at" db:"updated_at"` CurrentStatus string `json:"current_status" db:"current_status"`
BanExpiry *time.Time `json:"ban_expiry" db:"ban_expiry"`
CreatedAt time.Time `json:"created_at" db:"created_at"`
UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
} }
type AppealGuideline struct { type AppealGuideline struct {
ID uuid.UUID `json:"id" db:"id"` ID uuid.UUID `json:"id" db:"id"`
ViolationType string `json:"violation_type" db:"violation_type"` ViolationType string `json:"violation_type" db:"violation_type"`
MaxAppealsPerMonth int `json:"max_appeals_per_month" db:"max_appeals_per_month"` MaxAppealsPerMonth int `json:"max_appeals_per_month" db:"max_appeals_per_month"`
AppealWindowHours int `json:"appeal_window_hours" db:"appeal_window_hours"` AppealWindowHours int `json:"appeal_window_hours" db:"appeal_window_hours"`
AutoBanThreshold int `json:"auto_ban_threshold" db:"auto_ban_threshold"` AutoBanThreshold int `json:"auto_ban_threshold" db:"auto_ban_threshold"`
HardViolationBanThreshold int `json:"hard_violation_ban_threshold" db:"hard_violation_ban_threshold"` HardViolationBanThreshold int `json:"hard_violation_ban_threshold" db:"hard_violation_ban_threshold"`
IsActive bool `json:"is_active" db:"is_active"` IsActive bool `json:"is_active" db:"is_active"`
CreatedAt time.Time `json:"created_at" db:"created_at"` CreatedAt time.Time `json:"created_at" db:"created_at"`
UpdatedAt time.Time `json:"updated_at" db:"updated_at"` UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
} }
// DTOs for API responses // DTOs for API responses
type UserViolationResponse struct { type UserViolationResponse struct {
UserViolation UserViolation
FlagReason string `json:"flag_reason"` FlagReason string `json:"flag_reason"`
PostContent string `json:"post_content,omitempty"` PostContent string `json:"post_content,omitempty"`
CommentContent string `json:"comment_content,omitempty"` CommentContent string `json:"comment_content,omitempty"`
CanAppeal bool `json:"can_appeal"` CanAppeal bool `json:"can_appeal"`
AppealDeadline *time.Time `json:"appeal_deadline,omitempty"` AppealDeadline *time.Time `json:"appeal_deadline,omitempty"`
Appeal *UserAppeal `json:"appeal,omitempty"` Appeal *UserAppeal `json:"appeal,omitempty"`
} }
type UserAppealRequest struct { type UserAppealRequest struct {
UserViolationID uuid.UUID `json:"user_violation_id" binding:"required"` UserViolationID uuid.UUID `json:"user_violation_id" binding:"required"`
AppealReason string `json:"appeal_reason" binding:"required,min=10,max=1000"` AppealReason string `json:"appeal_reason" binding:"required,min=10,max=1000"`
AppealContext string `json:"appeal_context,omitempty" binding:"max=2000"` AppealContext string `json:"appeal_context,omitempty" binding:"max=2000"`
EvidenceURLs []string `json:"evidence_urls,omitempty"` EvidenceURLs []string `json:"evidence_urls,omitempty"`
} }
type UserAppealResponse struct { type UserAppealResponse struct {
@ -113,11 +130,11 @@ type UserAppealResponse struct {
} }
type UserViolationSummary struct { type UserViolationSummary struct {
TotalViolations int `json:"total_violations"` TotalViolations int `json:"total_violations"`
HardViolations int `json:"hard_violations"` HardViolations int `json:"hard_violations"`
SoftViolations int `json:"soft_violations"` SoftViolations int `json:"soft_violations"`
ActiveAppeals int `json:"active_appeals"` ActiveAppeals int `json:"active_appeals"`
CurrentStatus string `json:"current_status"` CurrentStatus string `json:"current_status"`
BanExpiry *time.Time `json:"ban_expiry,omitempty"` BanExpiry *time.Time `json:"ban_expiry,omitempty"`
RecentViolations []UserViolationResponse `json:"recent_violations"` RecentViolations []UserViolationResponse `json:"recent_violations"`
} }