feat: wire image + video thumbnail moderation into post creation flow (OpenRouter vision model, worst-outcome merge)
This commit is contained in:
parent
bc35eea69b
commit
77ef1ecac5
|
|
@ -342,18 +342,73 @@ func (h *PostHandler) CreatePost(c *gin.Context) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 5b. OpenRouter AI Moderation — NSFW vs Flag decision
|
// 5b. OpenRouter AI Moderation — NSFW vs Flag decision (text + image + video)
|
||||||
userSelfLabeledNSFW := req.IsNSFW
|
userSelfLabeledNSFW := req.IsNSFW
|
||||||
orDecision := ""
|
orDecision := ""
|
||||||
if h.openRouterService != nil {
|
if h.openRouterService != nil {
|
||||||
orResult, orErr := h.openRouterService.ModerateText(c.Request.Context(), req.Body)
|
// Collect all moderation results to pick the worst outcome
|
||||||
if orErr == nil && orResult != nil {
|
var allResults []*services.ModerationResult
|
||||||
orDecision = orResult.Action
|
|
||||||
switch orResult.Action {
|
// Text moderation
|
||||||
|
if req.Body != "" {
|
||||||
|
textResult, textErr := h.openRouterService.ModerateText(c.Request.Context(), req.Body)
|
||||||
|
if textErr == nil && textResult != nil {
|
||||||
|
allResults = append(allResults, textResult)
|
||||||
|
log.Info().Str("action", textResult.Action).Msg("Text moderation result")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Image moderation (vision model)
|
||||||
|
if req.ImageURL != nil && *req.ImageURL != "" {
|
||||||
|
imgResult, imgErr := h.openRouterService.ModerateImage(c.Request.Context(), *req.ImageURL)
|
||||||
|
if imgErr == nil && imgResult != nil {
|
||||||
|
allResults = append(allResults, imgResult)
|
||||||
|
log.Info().Str("action", imgResult.Action).Str("url", *req.ImageURL).Msg("Image moderation result")
|
||||||
|
} else if imgErr != nil {
|
||||||
|
log.Warn().Err(imgErr).Msg("Image moderation failed — continuing with text result only")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Video moderation via thumbnail (vision model on representative frame)
|
||||||
|
if req.VideoURL != nil && *req.VideoURL != "" {
|
||||||
|
thumbnailURL := ""
|
||||||
|
if req.Thumbnail != nil && *req.Thumbnail != "" {
|
||||||
|
thumbnailURL = *req.Thumbnail
|
||||||
|
}
|
||||||
|
if thumbnailURL != "" {
|
||||||
|
vidResult, vidErr := h.openRouterService.ModerateImage(c.Request.Context(), thumbnailURL)
|
||||||
|
if vidErr == nil && vidResult != nil {
|
||||||
|
allResults = append(allResults, vidResult)
|
||||||
|
log.Info().Str("action", vidResult.Action).Str("thumbnail", thumbnailURL).Msg("Video thumbnail moderation result")
|
||||||
|
} else if vidErr != nil {
|
||||||
|
log.Warn().Err(vidErr).Msg("Video thumbnail moderation failed — continuing without")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merge results: worst outcome wins (flag > nsfw > clean)
|
||||||
|
worstAction := "clean"
|
||||||
|
var worstResult *services.ModerationResult
|
||||||
|
for _, r := range allResults {
|
||||||
|
if r.Action == "flag" {
|
||||||
|
worstAction = "flag"
|
||||||
|
worstResult = r
|
||||||
|
break // Can't get worse than flag
|
||||||
|
} else if r.Action == "nsfw" && worstAction != "flag" {
|
||||||
|
worstAction = "nsfw"
|
||||||
|
worstResult = r
|
||||||
|
} else if worstResult == nil {
|
||||||
|
worstResult = r
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if worstResult != nil {
|
||||||
|
orDecision = worstAction
|
||||||
|
switch worstAction {
|
||||||
case "nsfw":
|
case "nsfw":
|
||||||
post.IsNSFW = true
|
post.IsNSFW = true
|
||||||
if orResult.NSFWReason != "" {
|
if worstResult.NSFWReason != "" {
|
||||||
post.NSFWReason = orResult.NSFWReason
|
post.NSFWReason = worstResult.NSFWReason
|
||||||
}
|
}
|
||||||
if post.Status != "pending_moderation" {
|
if post.Status != "pending_moderation" {
|
||||||
post.Status = "active" // NSFW posts are active but blurred
|
post.Status = "active" // NSFW posts are active but blurred
|
||||||
|
|
@ -362,9 +417,9 @@ func (h *PostHandler) CreatePost(c *gin.Context) {
|
||||||
// NOT ALLOWED — will be removed after creation
|
// NOT ALLOWED — will be removed after creation
|
||||||
post.Status = "removed"
|
post.Status = "removed"
|
||||||
}
|
}
|
||||||
// Update CIS from OpenRouter scores if available
|
// Update CIS from worst result scores if available
|
||||||
if orResult.Hate > 0 || orResult.Greed > 0 || orResult.Delusion > 0 {
|
if worstResult.Hate > 0 || worstResult.Greed > 0 || worstResult.Delusion > 0 {
|
||||||
orCis := 1.0 - (orResult.Hate+orResult.Greed+orResult.Delusion)/3.0
|
orCis := 1.0 - (worstResult.Hate+worstResult.Greed+worstResult.Delusion)/3.0
|
||||||
post.CISScore = &orCis
|
post.CISScore = &orCis
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue