80 lines
1.9 KiB
Go
80 lines
1.9 KiB
Go
package services
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
type ModerationService struct {
|
|
pool *pgxpool.Pool
|
|
// In a real app, we would have API keys here
|
|
}
|
|
|
|
func NewModerationService(pool *pgxpool.Pool) *ModerationService {
|
|
return &ModerationService{pool: pool}
|
|
}
|
|
|
|
type ThreePoisonsScore struct {
|
|
Hate float64 `json:"hate"`
|
|
Greed float64 `json:"greed"`
|
|
Delusion float64 `json:"delusion"`
|
|
}
|
|
|
|
func (s *ModerationService) AnalyzeContent(ctx context.Context, body string) (*ThreePoisonsScore, string, error) {
|
|
// PLACEHOLDER: In production, call OpenAI Moderation or HuggingFace
|
|
// For now, we use a simple rule-based mock to demonstrate the "Sanctuary" gates.
|
|
|
|
score := &ThreePoisonsScore{
|
|
Hate: 0.0,
|
|
Greed: 0.0,
|
|
Delusion: 0.0,
|
|
}
|
|
|
|
// Simple mock rules
|
|
if containsAny(body, []string{"hate", "kill", "attack"}) {
|
|
score.Hate = 0.8
|
|
}
|
|
if containsAny(body, []string{"buy", "crypto", "rich", "scam"}) {
|
|
score.Greed = 0.7
|
|
}
|
|
if containsAny(body, []string{"fake", "truth", "conspiracy"}) {
|
|
score.Delusion = 0.6
|
|
}
|
|
|
|
flagReason := ""
|
|
if score.Hate > 0.5 {
|
|
flagReason = "hate"
|
|
} else if score.Greed > 0.5 {
|
|
flagReason = "greed"
|
|
} else if score.Delusion > 0.5 {
|
|
flagReason = "delusion"
|
|
}
|
|
|
|
return score, flagReason, nil
|
|
}
|
|
|
|
func (s *ModerationService) FlagPost(ctx context.Context, postID uuid.UUID, scores *ThreePoisonsScore, reason string) error {
|
|
scoresJSON, _ := json.Marshal(scores)
|
|
query := `
|
|
INSERT INTO public.pending_moderation (post_id, flag_reason, scores)
|
|
VALUES ($1, $2, $3)
|
|
`
|
|
_, err := s.pool.Exec(ctx, query, postID, reason, scoresJSON)
|
|
return err
|
|
}
|
|
|
|
func containsAny(body string, terms []string) bool {
|
|
// Case insensitive check
|
|
lower := bytes.ToLower([]byte(body))
|
|
for _, term := range terms {
|
|
if bytes.Contains(lower, []byte(term)) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|