patrick/sojorn
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

feat: wire image + video thumbnail moderation into post creation flow (OpenRouter vision model, worst-outcome merge)

Browse source
This commit is contained in:
Patrick Britton 2026-02-07 18:21:36 -06:00
parent bc35eea69b
commit 77ef1ecac5
1 changed files with 65 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
go-backend/internal/handlers/post_handler.go
View file

@ -342,18 +342,73 @@ func (h *PostHandler) CreatePost(c *gin.Context) {
}
}
// 5b. OpenRouter AI Moderation — NSFW vs Flag decision
// 5b. OpenRouter AI Moderation — NSFW vs Flag decision (text + image + video)
userSelfLabeledNSFW := req.IsNSFW
orDecision := ""
if h.openRouterService != nil {
orResult, orErr := h.openRouterService.ModerateText(c.Request.Context(), req.Body)
if orErr == nil && orResult != nil {
orDecision = orResult.Action
switch orResult.Action {
// Collect all moderation results to pick the worst outcome
var allResults []*services.ModerationResult
// Text moderation
if req.Body != "" {
textResult, textErr := h.openRouterService.ModerateText(c.Request.Context(), req.Body)
if textErr == nil && textResult != nil {
allResults = append(allResults, textResult)
log.Info().Str("action", textResult.Action).Msg("Text moderation result")
}
}
// Image moderation (vision model)
if req.ImageURL != nil && *req.ImageURL != "" {
imgResult, imgErr := h.openRouterService.ModerateImage(c.Request.Context(), *req.ImageURL)
if imgErr == nil && imgResult != nil {
allResults = append(allResults, imgResult)
log.Info().Str("action", imgResult.Action).Str("url", *req.ImageURL).Msg("Image moderation result")
} else if imgErr != nil {
log.Warn().Err(imgErr).Msg("Image moderation failed — continuing with text result only")
}
}
// Video moderation via thumbnail (vision model on representative frame)
if req.VideoURL != nil && *req.VideoURL != "" {
thumbnailURL := ""
if req.Thumbnail != nil && *req.Thumbnail != "" {
thumbnailURL = *req.Thumbnail
}
if thumbnailURL != "" {
vidResult, vidErr := h.openRouterService.ModerateImage(c.Request.Context(), thumbnailURL)
if vidErr == nil && vidResult != nil {
allResults = append(allResults, vidResult)
log.Info().Str("action", vidResult.Action).Str("thumbnail", thumbnailURL).Msg("Video thumbnail moderation result")
} else if vidErr != nil {
log.Warn().Err(vidErr).Msg("Video thumbnail moderation failed — continuing without")
}
}
}
// Merge results: worst outcome wins (flag > nsfw > clean)
worstAction := "clean"
var worstResult *services.ModerationResult
for _, r := range allResults {
if r.Action == "flag" {
worstAction = "flag"
worstResult = r
break // Can't get worse than flag
} else if r.Action == "nsfw" && worstAction != "flag" {
worstAction = "nsfw"
worstResult = r
} else if worstResult == nil {
worstResult = r
}
}
if worstResult != nil {
orDecision = worstAction
switch worstAction {
case "nsfw":
post.IsNSFW = true
if orResult.NSFWReason != "" {
post.NSFWReason = orResult.NSFWReason
if worstResult.NSFWReason != "" {
post.NSFWReason = worstResult.NSFWReason
}
if post.Status != "pending_moderation" {
post.Status = "active" // NSFW posts are active but blurred
@ -362,9 +417,9 @@ func (h *PostHandler) CreatePost(c *gin.Context) {
// NOT ALLOWED — will be removed after creation
post.Status = "removed"
}
// Update CIS from OpenRouter scores if available
if orResult.Hate > 0 || orResult.Greed > 0 || orResult.Delusion > 0 {
orCis := 1.0 - (orResult.Hate+orResult.Greed+orResult.Delusion)/3.0
// Update CIS from worst result scores if available
if worstResult.Hate > 0 || worstResult.Greed > 0 || worstResult.Delusion > 0 {
orCis := 1.0 - (worstResult.Hate+worstResult.Greed+worstResult.Delusion)/3.0
post.CISScore = &orCis
}
}