AI moderation config: OpenRouter integration, admin console page, 10s quip limit
This commit is contained in:
Patrick Britton
parent
d73b73ac89
commit
7c52a1a1ed
412
admin/src/app/ai-moderation/page.tsx
Normal file
412
admin/src/app/ai-moderation/page.tsx
Normal file
|
|
@ -0,0 +1,412 @@
|
|||
'use client';
|
||||
|
||||
import AdminShell from '@/components/AdminShell';
|
||||
import { api } from '@/lib/api';
|
||||
import { useEffect, useState, useCallback, useRef } from 'react';
|
||||
import { Brain, Search, Check, Power, PowerOff, ChevronDown, Play, Loader2, Eye, MessageSquare, Video, Sparkles } from 'lucide-react';
|
||||
|
||||
const MODERATION_TYPES = [
|
||||
{ key: 'text', label: 'Text Moderation', icon: MessageSquare, desc: 'Analyze post text, comments, and captions for policy violations' },
|
||||
{ key: 'image', label: 'Image Moderation', icon: Eye, desc: 'Analyze uploaded images for inappropriate content (requires vision model)' },
|
||||
{ key: 'video', label: 'Video Moderation', icon: Video, desc: 'Analyze video frames extracted from Quips (requires vision model)' },
|
||||
];
|
||||
|
||||
interface ModelInfo {
|
||||
id: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
pricing: { prompt: string; completion: string; image?: string };
|
||||
context_length: number;
|
||||
architecture?: Record<string, any>;
|
||||
}
|
||||
|
||||
interface ModerationConfig {
|
||||
id: string;
|
||||
moderation_type: string;
|
||||
model_id: string;
|
||||
model_name: string;
|
||||
system_prompt: string;
|
||||
enabled: boolean;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export default function AIModerationPage() {
|
||||
const [configs, setConfigs] = useState<ModerationConfig[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [activeType, setActiveType] = useState('text');
|
||||
|
||||
const loadConfigs = useCallback(() => {
|
||||
setLoading(true);
|
||||
api.getAIModerationConfigs()
|
||||
.then((data) => setConfigs(data.configs || []))
|
||||
.catch(() => {})
|
||||
.finally(() => setLoading(false));
|
||||
}, []);
|
||||
|
||||
useEffect(() => { loadConfigs(); }, [loadConfigs]);
|
||||
|
||||
const getConfig = (type: string) => configs.find(c => c.moderation_type === type);
|
||||
|
||||
return (
|
||||
<AdminShell>
|
||||
<div className="mb-6">
|
||||
<h1 className="text-2xl font-bold text-gray-900 flex items-center gap-2">
|
||||
<Brain className="w-6 h-6" /> AI Moderation
|
||||
</h1>
|
||||
<p className="text-sm text-gray-500 mt-1">Configure AI models for content moderation via OpenRouter</p>
|
||||
</div>
|
||||
|
||||
{/* Config Cards */}
|
||||
<div className="grid grid-cols-1 md:grid-cols-3 gap-4 mb-6">
|
||||
{MODERATION_TYPES.map((mt) => {
|
||||
const config = getConfig(mt.key);
|
||||
const Icon = mt.icon;
|
||||
return (
|
||||
<button
|
||||
key={mt.key}
|
||||
onClick={() => setActiveType(mt.key)}
|
||||
className={`card p-4 text-left transition-all ${
|
||||
activeType === mt.key ? 'ring-2 ring-brand-500 shadow-md' : 'hover:shadow-sm'
|
||||
}`}
|
||||
>
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<div className="flex items-center gap-2">
|
||||
<Icon className="w-5 h-5 text-gray-600" />
|
||||
<span className="font-semibold text-gray-900 text-sm">{mt.label}</span>
|
||||
</div>
|
||||
{config?.enabled ? (
|
||||
<span className="flex items-center gap-1 text-xs font-medium text-green-700 bg-green-100 px-2 py-0.5 rounded-full">
|
||||
<Power className="w-3 h-3" /> On
|
||||
</span>
|
||||
) : (
|
||||
<span className="flex items-center gap-1 text-xs font-medium text-gray-400 bg-gray-100 px-2 py-0.5 rounded-full">
|
||||
<PowerOff className="w-3 h-3" /> Off
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<p className="text-xs text-gray-400 mb-2">{mt.desc}</p>
|
||||
{config?.model_id ? (
|
||||
<p className="text-xs font-mono text-brand-600 truncate">{config.model_name || config.model_id}</p>
|
||||
) : (
|
||||
<p className="text-xs text-gray-300 italic">No model selected</p>
|
||||
)}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
{/* Active Config Editor */}
|
||||
<ConfigEditor
|
||||
key={activeType}
|
||||
moderationType={activeType}
|
||||
config={getConfig(activeType)}
|
||||
onSaved={loadConfigs}
|
||||
/>
|
||||
</AdminShell>
|
||||
);
|
||||
}
|
||||
|
||||
// ─── Config Editor for a single moderation type ─────────
|
||||
|
||||
function ConfigEditor({ moderationType, config, onSaved }: {
|
||||
moderationType: string;
|
||||
config?: ModerationConfig;
|
||||
onSaved: () => void;
|
||||
}) {
|
||||
const [modelId, setModelId] = useState(config?.model_id || '');
|
||||
const [modelName, setModelName] = useState(config?.model_name || '');
|
||||
const [systemPrompt, setSystemPrompt] = useState(config?.system_prompt || '');
|
||||
const [enabled, setEnabled] = useState(config?.enabled || false);
|
||||
const [saving, setSaving] = useState(false);
|
||||
|
||||
// Model picker
|
||||
const [showPicker, setShowPicker] = useState(false);
|
||||
const [models, setModels] = useState<ModelInfo[]>([]);
|
||||
const [modelsLoading, setModelsLoading] = useState(false);
|
||||
const [searchTerm, setSearchTerm] = useState('');
|
||||
const [capability, setCapability] = useState('');
|
||||
const searchTimer = useRef<any>(null);
|
||||
|
||||
// Test
|
||||
const [testInput, setTestInput] = useState('');
|
||||
const [testResult, setTestResult] = useState<any>(null);
|
||||
const [testing, setTesting] = useState(false);
|
||||
|
||||
const loadModels = useCallback((search?: string, cap?: string) => {
|
||||
setModelsLoading(true);
|
||||
api.listOpenRouterModels({ search, capability: cap })
|
||||
.then((data) => setModels(data.models || []))
|
||||
.catch(() => {})
|
||||
.finally(() => setModelsLoading(false));
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (showPicker) loadModels(searchTerm || undefined, capability || undefined);
|
||||
}, [showPicker]);
|
||||
|
||||
const onSearchChange = (val: string) => {
|
||||
setSearchTerm(val);
|
||||
if (searchTimer.current) clearTimeout(searchTimer.current);
|
||||
searchTimer.current = setTimeout(() => {
|
||||
loadModels(val || undefined, capability || undefined);
|
||||
}, 400);
|
||||
};
|
||||
|
||||
const onCapabilityChange = (val: string) => {
|
||||
setCapability(val);
|
||||
loadModels(searchTerm || undefined, val || undefined);
|
||||
};
|
||||
|
||||
const selectModel = (m: ModelInfo) => {
|
||||
setModelId(m.id);
|
||||
setModelName(m.name);
|
||||
setShowPicker(false);
|
||||
};
|
||||
|
||||
const handleSave = async () => {
|
||||
setSaving(true);
|
||||
try {
|
||||
await api.setAIModerationConfig({
|
||||
moderation_type: moderationType,
|
||||
model_id: modelId,
|
||||
model_name: modelName,
|
||||
system_prompt: systemPrompt,
|
||||
enabled,
|
||||
});
|
||||
onSaved();
|
||||
} catch (e: any) {
|
||||
alert(e.message);
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleTest = async () => {
|
||||
if (!testInput.trim()) return;
|
||||
setTesting(true);
|
||||
setTestResult(null);
|
||||
try {
|
||||
const data = moderationType === 'text'
|
||||
? { moderation_type: moderationType, content: testInput }
|
||||
: { moderation_type: moderationType, image_url: testInput };
|
||||
const res = await api.testAIModeration(data);
|
||||
setTestResult(res.result);
|
||||
} catch (e: any) {
|
||||
setTestResult({ error: e.message });
|
||||
} finally {
|
||||
setTesting(false);
|
||||
}
|
||||
};
|
||||
|
||||
const isFree = (m: ModelInfo) => m.pricing.prompt === '0' || m.pricing.prompt === '0.0';
|
||||
const isVision = (m: ModelInfo) => {
|
||||
const modality = m.architecture?.modality;
|
||||
return typeof modality === 'string' && modality.includes('image');
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
{/* Model Selection */}
|
||||
<div className="card p-5">
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<h2 className="font-semibold text-gray-900">
|
||||
{MODERATION_TYPES.find(t => t.key === moderationType)?.label} Configuration
|
||||
</h2>
|
||||
<label className="flex items-center gap-2 cursor-pointer">
|
||||
<span className="text-sm text-gray-500">Enabled</span>
|
||||
<button
|
||||
onClick={() => setEnabled(!enabled)}
|
||||
className={`relative w-10 h-6 rounded-full transition-colors ${enabled ? 'bg-green-500' : 'bg-gray-300'}`}
|
||||
>
|
||||
<span className={`absolute top-0.5 w-5 h-5 bg-white rounded-full shadow transition-transform ${enabled ? 'left-[18px]' : 'left-0.5'}`} />
|
||||
</button>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{/* Selected Model */}
|
||||
<div className="mb-4">
|
||||
<label className="text-sm font-medium text-gray-600 block mb-1">Model</label>
|
||||
<div className="flex gap-2">
|
||||
<div
|
||||
onClick={() => setShowPicker(!showPicker)}
|
||||
className="flex-1 flex items-center justify-between px-3 py-2 border border-warm-300 rounded-lg cursor-pointer hover:bg-warm-50 transition-colors"
|
||||
>
|
||||
{modelId ? (
|
||||
<div>
|
||||
<span className="text-sm font-medium text-gray-900">{modelName || modelId}</span>
|
||||
<span className="text-xs text-gray-400 ml-2 font-mono">{modelId}</span>
|
||||
</div>
|
||||
) : (
|
||||
<span className="text-sm text-gray-400">Click to select a model...</span>
|
||||
)}
|
||||
<ChevronDown className="w-4 h-4 text-gray-400" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Model Picker */}
|
||||
{showPicker && (
|
||||
<div className="mb-4 border border-warm-200 rounded-lg overflow-hidden">
|
||||
<div className="p-3 bg-warm-100 flex gap-2">
|
||||
<div className="relative flex-1">
|
||||
<Search className="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-gray-400" />
|
||||
<input
|
||||
type="text"
|
||||
placeholder="Search models..."
|
||||
value={searchTerm}
|
||||
onChange={(e) => onSearchChange(e.target.value)}
|
||||
className="w-full pl-9 pr-3 py-2 text-sm border border-warm-300 rounded-lg focus:outline-none focus:ring-2 focus:ring-brand-500"
|
||||
/>
|
||||
</div>
|
||||
<select
|
||||
value={capability}
|
||||
onChange={(e) => onCapabilityChange(e.target.value)}
|
||||
className="text-sm border border-warm-300 rounded-lg px-3 py-2 focus:outline-none focus:ring-2 focus:ring-brand-500"
|
||||
>
|
||||
<option value="">All Models</option>
|
||||
<option value="free">Free Only</option>
|
||||
<option value="vision">Vision Only</option>
|
||||
</select>
|
||||
</div>
|
||||
<div className="max-h-80 overflow-y-auto">
|
||||
{modelsLoading ? (
|
||||
<div className="p-4 text-center text-gray-400 text-sm flex items-center justify-center gap-2">
|
||||
<Loader2 className="w-4 h-4 animate-spin" /> Loading models...
|
||||
</div>
|
||||
) : models.length === 0 ? (
|
||||
<div className="p-4 text-center text-gray-400 text-sm">No models found</div>
|
||||
) : (
|
||||
models.map((m) => (
|
||||
<div
|
||||
key={m.id}
|
||||
onClick={() => selectModel(m)}
|
||||
className={`px-3 py-2.5 border-b border-warm-100 cursor-pointer hover:bg-brand-50 transition-colors ${modelId === m.id ? 'bg-brand-50' : ''}`}
|
||||
>
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-2 min-w-0">
|
||||
{modelId === m.id && <Check className="w-4 h-4 text-brand-600 flex-shrink-0" />}
|
||||
<div className="min-w-0">
|
||||
<div className="text-sm font-medium text-gray-900 truncate">{m.name}</div>
|
||||
<div className="text-xs text-gray-400 font-mono truncate">{m.id}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-2 flex-shrink-0 ml-2">
|
||||
{isVision(m) && (
|
||||
<span className="text-xs bg-purple-100 text-purple-700 px-1.5 py-0.5 rounded font-medium">Vision</span>
|
||||
)}
|
||||
{isFree(m) ? (
|
||||
<span className="text-xs bg-green-100 text-green-700 px-1.5 py-0.5 rounded font-medium">Free</span>
|
||||
) : (
|
||||
<span className="text-xs text-gray-400">${m.pricing.prompt}/tok</span>
|
||||
)}
|
||||
<span className="text-xs text-gray-300">{(m.context_length / 1000).toFixed(0)}k ctx</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* System Prompt */}
|
||||
<div className="mb-4">
|
||||
<label className="text-sm font-medium text-gray-600 block mb-1">
|
||||
System Prompt <span className="text-gray-400 font-normal">(leave blank for default)</span>
|
||||
</label>
|
||||
<textarea
|
||||
rows={5}
|
||||
value={systemPrompt}
|
||||
onChange={(e) => setSystemPrompt(e.target.value)}
|
||||
placeholder="Custom system prompt for this moderation type... Leave blank to use the built-in default."
|
||||
className="w-full text-sm border border-warm-300 rounded-lg px-3 py-2 focus:outline-none focus:ring-2 focus:ring-brand-500 font-mono"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Save */}
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="text-xs text-gray-400">
|
||||
{config?.updated_at && `Last updated: ${new Date(config.updated_at).toLocaleString()}`}
|
||||
</div>
|
||||
<button onClick={handleSave} disabled={saving} className="btn-primary text-sm flex items-center gap-1.5">
|
||||
<Sparkles className="w-4 h-4" /> {saving ? 'Saving...' : 'Save Configuration'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Test Panel */}
|
||||
<div className="card p-5">
|
||||
<h3 className="font-semibold text-gray-900 mb-3 flex items-center gap-2">
|
||||
<Play className="w-4 h-4" /> Test Moderation
|
||||
</h3>
|
||||
<p className="text-xs text-gray-400 mb-3">
|
||||
{moderationType === 'text'
|
||||
? 'Enter text content to test moderation'
|
||||
: 'Enter an image URL to test vision moderation'}
|
||||
</p>
|
||||
<div className="flex gap-2 mb-3">
|
||||
<input
|
||||
type="text"
|
||||
value={testInput}
|
||||
onChange={(e) => setTestInput(e.target.value)}
|
||||
placeholder={moderationType === 'text' ? 'Enter test text...' : 'Enter image URL...'}
|
||||
className="flex-1 text-sm border border-warm-300 rounded-lg px-3 py-2 focus:outline-none focus:ring-2 focus:ring-brand-500"
|
||||
/>
|
||||
<button
|
||||
onClick={handleTest}
|
||||
disabled={testing || !modelId || !enabled}
|
||||
className="btn-primary text-sm flex items-center gap-1.5 disabled:opacity-40"
|
||||
>
|
||||
{testing ? <Loader2 className="w-4 h-4 animate-spin" /> : <Play className="w-4 h-4" />}
|
||||
Test
|
||||
</button>
|
||||
</div>
|
||||
{!modelId && <p className="text-xs text-amber-600">Select and save a model first to test</p>}
|
||||
|
||||
{testResult && (
|
||||
<div className={`p-3 rounded-lg text-sm ${testResult.error ? 'bg-red-50 text-red-700' : testResult.flagged ? 'bg-red-50' : 'bg-green-50'}`}>
|
||||
{testResult.error ? (
|
||||
<p>{testResult.error}</p>
|
||||
) : (
|
||||
<div>
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<span className={`font-bold ${testResult.flagged ? 'text-red-700' : 'text-green-700'}`}>
|
||||
{testResult.flagged ? 'FLAGGED' : 'CLEAN'}
|
||||
</span>
|
||||
{testResult.reason && <span className="text-gray-600">— {testResult.reason}</span>}
|
||||
</div>
|
||||
<div className="grid grid-cols-3 gap-2">
|
||||
<ScoreBar label="Hate" value={testResult.hate} />
|
||||
<ScoreBar label="Greed" value={testResult.greed} />
|
||||
<ScoreBar label="Delusion" value={testResult.delusion} />
|
||||
</div>
|
||||
{testResult.raw_content && (
|
||||
<details className="mt-2">
|
||||
<summary className="text-xs text-gray-400 cursor-pointer">Raw response</summary>
|
||||
<pre className="mt-1 text-xs bg-white p-2 rounded border border-warm-200 overflow-x-auto whitespace-pre-wrap">{testResult.raw_content}</pre>
|
||||
</details>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ScoreBar({ label, value }: { label: string; value: number }) {
|
||||
const pct = Math.round((value || 0) * 100);
|
||||
const color = pct > 50 ? 'bg-red-500' : pct > 25 ? 'bg-amber-400' : 'bg-green-400';
|
||||
return (
|
||||
<div>
|
||||
<div className="flex justify-between text-xs mb-0.5">
|
||||
<span className="text-gray-500">{label}</span>
|
||||
<span className="font-mono text-gray-700">{pct}%</span>
|
||||
</div>
|
||||
<div className="h-1.5 bg-gray-200 rounded-full overflow-hidden">
|
||||
<div className={`h-full ${color} rounded-full transition-all`} style={{ width: `${pct}%` }} />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -6,7 +6,7 @@ import { useAuth } from '@/lib/auth';
|
|||
import { cn } from '@/lib/utils';
|
||||
import {
|
||||
LayoutDashboard, Users, FileText, Shield, Scale, Flag,
|
||||
Settings, Activity, LogOut, ChevronLeft, ChevronRight, Sliders, FolderTree, HardDrive, AtSign,
|
||||
Settings, Activity, LogOut, ChevronLeft, ChevronRight, Sliders, FolderTree, HardDrive, AtSign, Brain,
|
||||
} from 'lucide-react';
|
||||
import { useState } from 'react';
|
||||
|
||||
|
|
@ -20,6 +20,7 @@ const navItems = [
|
|||
{ href: '/algorithm', label: 'Algorithm', icon: Sliders },
|
||||
{ href: '/categories', label: 'Categories', icon: FolderTree },
|
||||
{ href: '/usernames', label: 'Usernames', icon: AtSign },
|
||||
{ href: '/ai-moderation', label: 'AI Moderation', icon: Brain },
|
||||
{ href: '/storage', label: 'Storage', icon: HardDrive },
|
||||
{ href: '/system', label: 'System Health', icon: Activity },
|
||||
{ href: '/settings', label: 'Settings', icon: Settings },
|
||||
|
|
|
|||
|
|
@ -326,6 +326,32 @@ class ApiClient {
|
|||
body: JSON.stringify({ decision, notes }),
|
||||
});
|
||||
}
|
||||
|
||||
// AI Moderation
|
||||
async listOpenRouterModels(params: { capability?: string; search?: string } = {}) {
|
||||
const qs = new URLSearchParams();
|
||||
if (params.capability) qs.set('capability', params.capability);
|
||||
if (params.search) qs.set('search', params.search);
|
||||
return this.request<any>(`/api/v1/admin/ai/models?${qs}`);
|
||||
}
|
||||
|
||||
async getAIModerationConfigs() {
|
||||
return this.request<any>('/api/v1/admin/ai/config');
|
||||
}
|
||||
|
||||
async setAIModerationConfig(data: { moderation_type: string; model_id: string; model_name: string; system_prompt: string; enabled: boolean }) {
|
||||
return this.request<any>('/api/v1/admin/ai/config', {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify(data),
|
||||
});
|
||||
}
|
||||
|
||||
async testAIModeration(data: { moderation_type: string; content?: string; image_url?: string }) {
|
||||
return this.request<any>('/api/v1/admin/ai/test', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(data),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export const api = new ApiClient();
|
||||
|
|
|
|||
|
|
@ -121,6 +121,9 @@ func main() {
|
|||
// Initialize appeal service
|
||||
appealService := services.NewAppealService(dbPool)
|
||||
|
||||
// Initialize OpenRouter service
|
||||
openRouterService := services.NewOpenRouterService(dbPool, cfg.OpenRouterAPIKey)
|
||||
|
||||
// Initialize content filter (hard blocklist + strike system)
|
||||
contentFilter := services.NewContentFilter(dbPool)
|
||||
|
||||
|
|
@ -155,7 +158,7 @@ func main() {
|
|||
}
|
||||
}
|
||||
|
||||
adminHandler := handlers.NewAdminHandler(dbPool, moderationService, appealService, emailService, cfg.JWTSecret, cfg.TurnstileSecretKey, s3Client, cfg.R2MediaBucket, cfg.R2VideoBucket, cfg.R2ImgDomain, cfg.R2VidDomain)
|
||||
adminHandler := handlers.NewAdminHandler(dbPool, moderationService, appealService, emailService, openRouterService, cfg.JWTSecret, cfg.TurnstileSecretKey, s3Client, cfg.R2MediaBucket, cfg.R2VideoBucket, cfg.R2ImgDomain, cfg.R2VidDomain)
|
||||
|
||||
mediaHandler := handlers.NewMediaHandler(
|
||||
s3Client,
|
||||
|
|
@ -443,6 +446,12 @@ func main() {
|
|||
// Username Claim Requests
|
||||
admin.GET("/usernames/claims", adminHandler.ListClaimRequests)
|
||||
admin.PATCH("/usernames/claims/:id", adminHandler.ReviewClaimRequest)
|
||||
|
||||
// AI Moderation Config
|
||||
admin.GET("/ai/models", adminHandler.ListOpenRouterModels)
|
||||
admin.GET("/ai/config", adminHandler.GetAIModerationConfigs)
|
||||
admin.PUT("/ai/config", adminHandler.SetAIModerationConfig)
|
||||
admin.POST("/ai/test", adminHandler.TestAIModeration)
|
||||
}
|
||||
|
||||
// Public claim request endpoint (no auth)
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ type Config struct {
|
|||
TurnstileSecretKey string
|
||||
APIBaseURL string
|
||||
AppBaseURL string
|
||||
OpenRouterAPIKey string
|
||||
}
|
||||
|
||||
func LoadConfig() *Config {
|
||||
|
|
@ -82,6 +83,7 @@ func LoadConfig() *Config {
|
|||
TurnstileSecretKey: getEnv("TURNSTILE_SECRET", ""),
|
||||
APIBaseURL: getEnv("API_BASE_URL", "https://api.sojorn.net"),
|
||||
AppBaseURL: getEnv("APP_BASE_URL", "https://sojorn.net"),
|
||||
OpenRouterAPIKey: getEnv("OPENROUTER_API", ""),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ type AdminHandler struct {
|
|||
moderationService *services.ModerationService
|
||||
appealService *services.AppealService
|
||||
emailService *services.EmailService
|
||||
openRouterService *services.OpenRouterService
|
||||
jwtSecret string
|
||||
turnstileSecret string
|
||||
s3Client *s3.Client
|
||||
|
|
@ -34,12 +35,13 @@ type AdminHandler struct {
|
|||
vidDomain string
|
||||
}
|
||||
|
||||
func NewAdminHandler(pool *pgxpool.Pool, moderationService *services.ModerationService, appealService *services.AppealService, emailService *services.EmailService, jwtSecret string, turnstileSecret string, s3Client *s3.Client, mediaBucket string, videoBucket string, imgDomain string, vidDomain string) *AdminHandler {
|
||||
func NewAdminHandler(pool *pgxpool.Pool, moderationService *services.ModerationService, appealService *services.AppealService, emailService *services.EmailService, openRouterService *services.OpenRouterService, jwtSecret string, turnstileSecret string, s3Client *s3.Client, mediaBucket string, videoBucket string, imgDomain string, vidDomain string) *AdminHandler {
|
||||
return &AdminHandler{
|
||||
pool: pool,
|
||||
moderationService: moderationService,
|
||||
appealService: appealService,
|
||||
emailService: emailService,
|
||||
openRouterService: openRouterService,
|
||||
jwtSecret: jwtSecret,
|
||||
turnstileSecret: turnstileSecret,
|
||||
s3Client: s3Client,
|
||||
|
|
@ -2304,3 +2306,152 @@ func (h *AdminHandler) SubmitClaimRequest(c *gin.Context) {
|
|||
|
||||
c.JSON(http.StatusOK, gin.H{"message": "Your claim request has been submitted and will be reviewed by our team."})
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// AI Moderation Config
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
func (h *AdminHandler) ListOpenRouterModels(c *gin.Context) {
|
||||
if h.openRouterService == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "OpenRouter not configured"})
|
||||
return
|
||||
}
|
||||
|
||||
models, err := h.openRouterService.ListModels(c.Request.Context())
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to fetch models: %v", err)})
|
||||
return
|
||||
}
|
||||
|
||||
// Optional filter by capability
|
||||
capability := c.Query("capability") // "text", "image", "vision", "free"
|
||||
if capability != "" {
|
||||
var filtered []services.OpenRouterModel
|
||||
for _, m := range models {
|
||||
switch capability {
|
||||
case "free":
|
||||
if m.Pricing.Prompt == "0" || m.Pricing.Prompt == "0.0" {
|
||||
filtered = append(filtered, m)
|
||||
}
|
||||
case "vision", "image":
|
||||
if arch, ok := m.Architecture["modality"].(string); ok {
|
||||
if strings.Contains(arch, "image") {
|
||||
filtered = append(filtered, m)
|
||||
}
|
||||
}
|
||||
default:
|
||||
filtered = append(filtered, m)
|
||||
}
|
||||
}
|
||||
models = filtered
|
||||
}
|
||||
|
||||
// Search filter
|
||||
search := strings.ToLower(c.Query("search"))
|
||||
if search != "" {
|
||||
var filtered []services.OpenRouterModel
|
||||
for _, m := range models {
|
||||
if strings.Contains(strings.ToLower(m.ID), search) || strings.Contains(strings.ToLower(m.Name), search) {
|
||||
filtered = append(filtered, m)
|
||||
}
|
||||
}
|
||||
models = filtered
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"models": models, "total": len(models)})
|
||||
}
|
||||
|
||||
func (h *AdminHandler) GetAIModerationConfigs(c *gin.Context) {
|
||||
if h.openRouterService == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "OpenRouter not configured"})
|
||||
return
|
||||
}
|
||||
|
||||
configs, err := h.openRouterService.GetModerationConfigs(c.Request.Context())
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to fetch configs: %v", err)})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"configs": configs})
|
||||
}
|
||||
|
||||
func (h *AdminHandler) SetAIModerationConfig(c *gin.Context) {
|
||||
if h.openRouterService == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "OpenRouter not configured"})
|
||||
return
|
||||
}
|
||||
|
||||
var req struct {
|
||||
ModerationType string `json:"moderation_type" binding:"required"`
|
||||
ModelID string `json:"model_id"`
|
||||
ModelName string `json:"model_name"`
|
||||
SystemPrompt string `json:"system_prompt"`
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
if req.ModerationType != "text" && req.ModerationType != "image" && req.ModerationType != "video" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "moderation_type must be text, image, or video"})
|
||||
return
|
||||
}
|
||||
|
||||
adminID := c.GetString("user_id")
|
||||
err := h.openRouterService.SetModerationConfig(c.Request.Context(), req.ModerationType, req.ModelID, req.ModelName, req.SystemPrompt, req.Enabled, adminID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to save config: %v", err)})
|
||||
return
|
||||
}
|
||||
|
||||
// Audit log
|
||||
h.pool.Exec(c.Request.Context(), `
|
||||
INSERT INTO audit_log (admin_id, action, target_type, details)
|
||||
VALUES ($1, 'update_ai_moderation', 'ai_config', $2)
|
||||
`, adminID, fmt.Sprintf("Set %s moderation model to %s (enabled=%v)", req.ModerationType, req.ModelID, req.Enabled))
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"message": "Configuration updated"})
|
||||
}
|
||||
|
||||
func (h *AdminHandler) TestAIModeration(c *gin.Context) {
|
||||
if h.openRouterService == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "OpenRouter not configured"})
|
||||
return
|
||||
}
|
||||
|
||||
var req struct {
|
||||
ModerationType string `json:"moderation_type" binding:"required"`
|
||||
Content string `json:"content"`
|
||||
ImageURL string `json:"image_url"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
ctx := c.Request.Context()
|
||||
var result *services.ModerationResult
|
||||
var err error
|
||||
|
||||
switch req.ModerationType {
|
||||
case "text":
|
||||
result, err = h.openRouterService.ModerateText(ctx, req.Content)
|
||||
case "image":
|
||||
result, err = h.openRouterService.ModerateImage(ctx, req.ImageURL)
|
||||
case "video":
|
||||
urls := strings.Split(req.ImageURL, ",")
|
||||
result, err = h.openRouterService.ModerateVideo(ctx, urls)
|
||||
default:
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid moderation_type"})
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"result": result})
|
||||
}
|
||||
|
|
|
|||
366
go-backend/internal/services/openrouter_service.go
Normal file
366
go-backend/internal/services/openrouter_service.go
Normal file
|
|
@ -0,0 +1,366 @@
|
|||
package services
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// OpenRouterService handles interactions with the OpenRouter API
|
||||
type OpenRouterService struct {
|
||||
pool *pgxpool.Pool
|
||||
httpClient *http.Client
|
||||
apiKey string
|
||||
|
||||
// Cached model list
|
||||
modelCache []OpenRouterModel
|
||||
modelCacheMu sync.RWMutex
|
||||
modelCacheTime time.Time
|
||||
}
|
||||
|
||||
// OpenRouterModel represents a model available on OpenRouter
|
||||
type OpenRouterModel struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Pricing OpenRouterPricing `json:"pricing"`
|
||||
ContextLength int `json:"context_length"`
|
||||
Architecture map[string]any `json:"architecture,omitempty"`
|
||||
TopProvider map[string]any `json:"top_provider,omitempty"`
|
||||
PerRequestLimits map[string]any `json:"per_request_limits,omitempty"`
|
||||
}
|
||||
|
||||
type OpenRouterPricing struct {
|
||||
Prompt string `json:"prompt"`
|
||||
Completion string `json:"completion"`
|
||||
Image string `json:"image,omitempty"`
|
||||
Request string `json:"request,omitempty"`
|
||||
}
|
||||
|
||||
// ModerationConfigEntry represents a row in ai_moderation_config
|
||||
type ModerationConfigEntry struct {
|
||||
ID string `json:"id"`
|
||||
ModerationType string `json:"moderation_type"`
|
||||
ModelID string `json:"model_id"`
|
||||
ModelName string `json:"model_name"`
|
||||
SystemPrompt string `json:"system_prompt"`
|
||||
Enabled bool `json:"enabled"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
UpdatedBy *string `json:"updated_by,omitempty"`
|
||||
}
|
||||
|
||||
// OpenRouterChatMessage represents a message in a chat completion request
|
||||
type OpenRouterChatMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content any `json:"content"`
|
||||
}
|
||||
|
||||
// OpenRouterChatRequest represents a chat completion request
|
||||
type OpenRouterChatRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []OpenRouterChatMessage `json:"messages"`
|
||||
}
|
||||
|
||||
// OpenRouterChatResponse represents a chat completion response
|
||||
type OpenRouterChatResponse struct {
|
||||
ID string `json:"id"`
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
} `json:"choices"`
|
||||
Usage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
} `json:"usage"`
|
||||
}
|
||||
|
||||
func NewOpenRouterService(pool *pgxpool.Pool, apiKey string) *OpenRouterService {
|
||||
return &OpenRouterService{
|
||||
pool: pool,
|
||||
apiKey: apiKey,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 60 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ListModels fetches available models from OpenRouter, with 1-hour cache
|
||||
func (s *OpenRouterService) ListModels(ctx context.Context) ([]OpenRouterModel, error) {
|
||||
s.modelCacheMu.RLock()
|
||||
if len(s.modelCache) > 0 && time.Since(s.modelCacheTime) < time.Hour {
|
||||
cached := s.modelCache
|
||||
s.modelCacheMu.RUnlock()
|
||||
return cached, nil
|
||||
}
|
||||
s.modelCacheMu.RUnlock()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://openrouter.ai/api/v1/models", nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
if s.apiKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+s.apiKey)
|
||||
}
|
||||
|
||||
resp, err := s.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch models: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("OpenRouter API error %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Data []OpenRouterModel `json:"data"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode models: %w", err)
|
||||
}
|
||||
|
||||
s.modelCacheMu.Lock()
|
||||
s.modelCache = result.Data
|
||||
s.modelCacheTime = time.Now()
|
||||
s.modelCacheMu.Unlock()
|
||||
|
||||
return result.Data, nil
|
||||
}
|
||||
|
||||
// GetModerationConfigs returns all moderation type configurations
|
||||
func (s *OpenRouterService) GetModerationConfigs(ctx context.Context) ([]ModerationConfigEntry, error) {
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT id, moderation_type, model_id, model_name, system_prompt, enabled, updated_at, updated_by
|
||||
FROM ai_moderation_config
|
||||
ORDER BY moderation_type
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query configs: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var configs []ModerationConfigEntry
|
||||
for rows.Next() {
|
||||
var c ModerationConfigEntry
|
||||
if err := rows.Scan(&c.ID, &c.ModerationType, &c.ModelID, &c.ModelName, &c.SystemPrompt, &c.Enabled, &c.UpdatedAt, &c.UpdatedBy); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
configs = append(configs, c)
|
||||
}
|
||||
return configs, nil
|
||||
}
|
||||
|
||||
// GetModerationConfig returns config for a specific moderation type
|
||||
func (s *OpenRouterService) GetModerationConfig(ctx context.Context, moderationType string) (*ModerationConfigEntry, error) {
|
||||
var c ModerationConfigEntry
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT id, moderation_type, model_id, model_name, system_prompt, enabled, updated_at, updated_by
|
||||
FROM ai_moderation_config WHERE moderation_type = $1
|
||||
`, moderationType).Scan(&c.ID, &c.ModerationType, &c.ModelID, &c.ModelName, &c.SystemPrompt, &c.Enabled, &c.UpdatedAt, &c.UpdatedBy)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// SetModerationConfig upserts a moderation config
|
||||
func (s *OpenRouterService) SetModerationConfig(ctx context.Context, moderationType, modelID, modelName, systemPrompt string, enabled bool, updatedBy string) error {
|
||||
_, err := s.pool.Exec(ctx, `
|
||||
INSERT INTO ai_moderation_config (moderation_type, model_id, model_name, system_prompt, enabled, updated_by, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, NOW())
|
||||
ON CONFLICT (moderation_type)
|
||||
DO UPDATE SET model_id = $2, model_name = $3, system_prompt = $4, enabled = $5, updated_by = $6, updated_at = NOW()
|
||||
`, moderationType, modelID, modelName, systemPrompt, enabled, updatedBy)
|
||||
return err
|
||||
}
|
||||
|
||||
// ModerateText sends text content to the configured model for moderation
|
||||
func (s *OpenRouterService) ModerateText(ctx context.Context, content string) (*ModerationResult, error) {
|
||||
config, err := s.GetModerationConfig(ctx, "text")
|
||||
if err != nil || !config.Enabled || config.ModelID == "" {
|
||||
return nil, fmt.Errorf("text moderation not configured")
|
||||
}
|
||||
return s.callModel(ctx, config.ModelID, config.SystemPrompt, content, nil)
|
||||
}
|
||||
|
||||
// ModerateImage sends an image URL to a vision model for moderation
|
||||
func (s *OpenRouterService) ModerateImage(ctx context.Context, imageURL string) (*ModerationResult, error) {
|
||||
config, err := s.GetModerationConfig(ctx, "image")
|
||||
if err != nil || !config.Enabled || config.ModelID == "" {
|
||||
return nil, fmt.Errorf("image moderation not configured")
|
||||
}
|
||||
return s.callModel(ctx, config.ModelID, config.SystemPrompt, "", []string{imageURL})
|
||||
}
|
||||
|
||||
// ModerateVideo sends video frame URLs to a vision model for moderation
|
||||
func (s *OpenRouterService) ModerateVideo(ctx context.Context, frameURLs []string) (*ModerationResult, error) {
|
||||
config, err := s.GetModerationConfig(ctx, "video")
|
||||
if err != nil || !config.Enabled || config.ModelID == "" {
|
||||
return nil, fmt.Errorf("video moderation not configured")
|
||||
}
|
||||
return s.callModel(ctx, config.ModelID, config.SystemPrompt, "These are 3 frames extracted from a short video. Analyze all frames for policy violations.", frameURLs)
|
||||
}
|
||||
|
||||
// ModerationResult is the parsed response from OpenRouter moderation
|
||||
type ModerationResult struct {
|
||||
Flagged bool `json:"flagged"`
|
||||
Reason string `json:"reason"`
|
||||
Hate float64 `json:"hate"`
|
||||
Greed float64 `json:"greed"`
|
||||
Delusion float64 `json:"delusion"`
|
||||
RawContent string `json:"raw_content"`
|
||||
}
|
||||
|
||||
// callModel sends a chat completion request to OpenRouter
|
||||
func (s *OpenRouterService) callModel(ctx context.Context, modelID, systemPrompt, textContent string, imageURLs []string) (*ModerationResult, error) {
|
||||
if s.apiKey == "" {
|
||||
return nil, fmt.Errorf("OpenRouter API key not configured")
|
||||
}
|
||||
|
||||
messages := []OpenRouterChatMessage{}
|
||||
|
||||
// System prompt
|
||||
if systemPrompt == "" {
|
||||
systemPrompt = defaultModerationSystemPrompt
|
||||
}
|
||||
messages = append(messages, OpenRouterChatMessage{Role: "system", Content: systemPrompt})
|
||||
|
||||
// User message — text only or multimodal (text + images)
|
||||
if len(imageURLs) > 0 {
|
||||
// Multimodal content array
|
||||
parts := []map[string]any{}
|
||||
if textContent != "" {
|
||||
parts = append(parts, map[string]any{"type": "text", "text": textContent})
|
||||
}
|
||||
for _, url := range imageURLs {
|
||||
parts = append(parts, map[string]any{
|
||||
"type": "image_url",
|
||||
"image_url": map[string]string{"url": url},
|
||||
})
|
||||
}
|
||||
messages = append(messages, OpenRouterChatMessage{Role: "user", Content: parts})
|
||||
} else {
|
||||
messages = append(messages, OpenRouterChatMessage{Role: "user", Content: textContent})
|
||||
}
|
||||
|
||||
reqBody := OpenRouterChatRequest{
|
||||
Model: modelID,
|
||||
Messages: messages,
|
||||
}
|
||||
|
||||
jsonBody, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://openrouter.ai/api/v1/chat/completions", bytes.NewBuffer(jsonBody))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+s.apiKey)
|
||||
req.Header.Set("HTTP-Referer", "https://sojorn.net")
|
||||
req.Header.Set("X-Title", "Sojorn Moderation")
|
||||
|
||||
resp, err := s.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("OpenRouter request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("OpenRouter API error %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var chatResp OpenRouterChatResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&chatResp); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode response: %w", err)
|
||||
}
|
||||
|
||||
if len(chatResp.Choices) == 0 {
|
||||
return nil, fmt.Errorf("no response from model")
|
||||
}
|
||||
|
||||
raw := chatResp.Choices[0].Message.Content
|
||||
return parseModerationResponse(raw), nil
|
||||
}
|
||||
|
||||
// parseModerationResponse tries to extract structured moderation data from model output
|
||||
func parseModerationResponse(raw string) *ModerationResult {
|
||||
result := &ModerationResult{RawContent: raw}
|
||||
|
||||
// Try to parse JSON from the response
|
||||
// Models may wrap JSON in markdown code blocks
|
||||
cleaned := raw
|
||||
if idx := strings.Index(cleaned, "```json"); idx >= 0 {
|
||||
cleaned = cleaned[idx+7:]
|
||||
if end := strings.Index(cleaned, "```"); end >= 0 {
|
||||
cleaned = cleaned[:end]
|
||||
}
|
||||
} else if idx := strings.Index(cleaned, "```"); idx >= 0 {
|
||||
cleaned = cleaned[idx+3:]
|
||||
if end := strings.Index(cleaned, "```"); end >= 0 {
|
||||
cleaned = cleaned[:end]
|
||||
}
|
||||
}
|
||||
cleaned = strings.TrimSpace(cleaned)
|
||||
|
||||
var parsed struct {
|
||||
Flagged bool `json:"flagged"`
|
||||
Reason string `json:"reason"`
|
||||
Hate float64 `json:"hate"`
|
||||
Greed float64 `json:"greed"`
|
||||
Delusion float64 `json:"delusion"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(cleaned), &parsed); err == nil {
|
||||
result.Flagged = parsed.Flagged
|
||||
result.Reason = parsed.Reason
|
||||
result.Hate = parsed.Hate
|
||||
result.Greed = parsed.Greed
|
||||
result.Delusion = parsed.Delusion
|
||||
return result
|
||||
}
|
||||
|
||||
// Fallback: check for keywords in raw text
|
||||
lower := strings.ToLower(raw)
|
||||
if strings.Contains(lower, "violation") || strings.Contains(lower, "inappropriate") || strings.Contains(lower, "flagged") {
|
||||
result.Flagged = true
|
||||
result.Reason = "Content flagged by AI moderation"
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
const defaultModerationSystemPrompt = `You are a content moderation AI for Sojorn, a social media platform.
|
||||
Analyze the provided content for policy violations.
|
||||
|
||||
Respond ONLY with a JSON object in this exact format:
|
||||
{
|
||||
"flagged": true/false,
|
||||
"reason": "brief reason if flagged, empty string if not",
|
||||
"hate": 0.0-1.0,
|
||||
"greed": 0.0-1.0,
|
||||
"delusion": 0.0-1.0
|
||||
}
|
||||
|
||||
Scoring guide (Three Poisons framework):
|
||||
- hate: harassment, threats, violence, sexual content, hate speech, discrimination
|
||||
- greed: spam, scams, crypto schemes, misleading promotions, get-rich-quick
|
||||
- delusion: misinformation, self-harm content, conspiracy theories, dangerous medical advice
|
||||
|
||||
Score 0.0 = no concern, 1.0 = extreme violation. Flag if any score > 0.5.
|
||||
Only respond with the JSON, no other text.`
|
||||
|
|
@ -12,7 +12,7 @@ import '../../../services/image_upload_service.dart';
|
|||
import '../../../theme/app_theme.dart';
|
||||
import '../../compose/video_editor_screen.dart';
|
||||
|
||||
/// Quip editor for videos with 15-second maximum duration.
|
||||
/// Quip editor for videos with 10-second maximum duration.
|
||||
/// Uses the standardized sojornVideoEditor and uploads via ImageUploadService.
|
||||
class QuipEditorScreen extends ConsumerStatefulWidget {
|
||||
final File videoFile;
|
||||
|
|
@ -31,7 +31,7 @@ class QuipEditorScreen extends ConsumerStatefulWidget {
|
|||
}
|
||||
|
||||
class _QuipEditorScreenState extends ConsumerState<QuipEditorScreen> {
|
||||
static const Duration _maxDuration = Duration(seconds: 15);
|
||||
static const Duration _maxDuration = Duration(seconds: 10);
|
||||
final TextEditingController _captionController = TextEditingController();
|
||||
final ImageUploadService _uploadService = ImageUploadService();
|
||||
|
||||
|
|
@ -167,7 +167,7 @@ class _QuipEditorScreenState extends ConsumerState<QuipEditorScreen> {
|
|||
padding: const EdgeInsets.only(right: 16),
|
||||
child: Center(
|
||||
child: Text(
|
||||
'Trim required (max 15s)',
|
||||
'Trim required (max 10s)',
|
||||
style: TextStyle(
|
||||
color: AppTheme.error,
|
||||
fontSize: 12,
|
||||
|
|
@ -211,7 +211,7 @@ class _QuipEditorScreenState extends ConsumerState<QuipEditorScreen> {
|
|||
if (!videoDurationOk) ...[
|
||||
const SizedBox(height: 8),
|
||||
Text(
|
||||
'Video exceeds 15s - trim required',
|
||||
'Video exceeds 10s - trim required',
|
||||
style: TextStyle(
|
||||
color: AppTheme.error,
|
||||
fontSize: 12,
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ class QuipRecorderScreen extends StatefulWidget {
|
|||
class _QuipRecorderScreenState extends State<QuipRecorderScreen>
|
||||
with WidgetsBindingObserver {
|
||||
// Config
|
||||
static const Duration _maxDuration = Duration(seconds: 60);
|
||||
static const Duration _maxDuration = Duration(seconds: 10);
|
||||
|
||||
// Camera State
|
||||
CameraController? _cameraController;
|
||||
|
|
@ -222,7 +222,7 @@ class _QuipRecorderScreenState extends State<QuipRecorderScreen>
|
|||
|
||||
Future<void> _pickFromGallery() async {
|
||||
final picker = ImagePicker();
|
||||
final video = await picker.pickVideo(source: ImageSource.gallery, maxDuration: _maxDuration);
|
||||
final video = await picker.pickVideo(source: ImageSource.gallery, maxDuration: const Duration(seconds: 10));
|
||||
if (video != null) {
|
||||
if (mounted) {
|
||||
Navigator.push(
|
||||
|
|
|
|||
Loading…
Reference in a new issue