AI moderation config: OpenRouter integration, admin console page, 10s quip limit
This commit is contained in:
Patrick Britton
parent
d73b73ac89
commit
7c52a1a1ed
412
admin/src/app/ai-moderation/page.tsx
Normal file
412
admin/src/app/ai-moderation/page.tsx
Normal file
|
|
@ -0,0 +1,412 @@
|
||||||
|
'use client';
|
||||||
|
|
||||||
|
import AdminShell from '@/components/AdminShell';
|
||||||
|
import { api } from '@/lib/api';
|
||||||
|
import { useEffect, useState, useCallback, useRef } from 'react';
|
||||||
|
import { Brain, Search, Check, Power, PowerOff, ChevronDown, Play, Loader2, Eye, MessageSquare, Video, Sparkles } from 'lucide-react';
|
||||||
|
|
||||||
|
const MODERATION_TYPES = [
|
||||||
|
{ key: 'text', label: 'Text Moderation', icon: MessageSquare, desc: 'Analyze post text, comments, and captions for policy violations' },
|
||||||
|
{ key: 'image', label: 'Image Moderation', icon: Eye, desc: 'Analyze uploaded images for inappropriate content (requires vision model)' },
|
||||||
|
{ key: 'video', label: 'Video Moderation', icon: Video, desc: 'Analyze video frames extracted from Quips (requires vision model)' },
|
||||||
|
];
|
||||||
|
|
||||||
|
interface ModelInfo {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
description?: string;
|
||||||
|
pricing: { prompt: string; completion: string; image?: string };
|
||||||
|
context_length: number;
|
||||||
|
architecture?: Record<string, any>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ModerationConfig {
|
||||||
|
id: string;
|
||||||
|
moderation_type: string;
|
||||||
|
model_id: string;
|
||||||
|
model_name: string;
|
||||||
|
system_prompt: string;
|
||||||
|
enabled: boolean;
|
||||||
|
updated_at: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function AIModerationPage() {
|
||||||
|
const [configs, setConfigs] = useState<ModerationConfig[]>([]);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [activeType, setActiveType] = useState('text');
|
||||||
|
|
||||||
|
const loadConfigs = useCallback(() => {
|
||||||
|
setLoading(true);
|
||||||
|
api.getAIModerationConfigs()
|
||||||
|
.then((data) => setConfigs(data.configs || []))
|
||||||
|
.catch(() => {})
|
||||||
|
.finally(() => setLoading(false));
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => { loadConfigs(); }, [loadConfigs]);
|
||||||
|
|
||||||
|
const getConfig = (type: string) => configs.find(c => c.moderation_type === type);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<AdminShell>
|
||||||
|
<div className="mb-6">
|
||||||
|
<h1 className="text-2xl font-bold text-gray-900 flex items-center gap-2">
|
||||||
|
<Brain className="w-6 h-6" /> AI Moderation
|
||||||
|
</h1>
|
||||||
|
<p className="text-sm text-gray-500 mt-1">Configure AI models for content moderation via OpenRouter</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Config Cards */}
|
||||||
|
<div className="grid grid-cols-1 md:grid-cols-3 gap-4 mb-6">
|
||||||
|
{MODERATION_TYPES.map((mt) => {
|
||||||
|
const config = getConfig(mt.key);
|
||||||
|
const Icon = mt.icon;
|
||||||
|
return (
|
||||||
|
<button
|
||||||
|
key={mt.key}
|
||||||
|
onClick={() => setActiveType(mt.key)}
|
||||||
|
className={`card p-4 text-left transition-all ${
|
||||||
|
activeType === mt.key ? 'ring-2 ring-brand-500 shadow-md' : 'hover:shadow-sm'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<div className="flex items-center justify-between mb-2">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<Icon className="w-5 h-5 text-gray-600" />
|
||||||
|
<span className="font-semibold text-gray-900 text-sm">{mt.label}</span>
|
||||||
|
</div>
|
||||||
|
{config?.enabled ? (
|
||||||
|
<span className="flex items-center gap-1 text-xs font-medium text-green-700 bg-green-100 px-2 py-0.5 rounded-full">
|
||||||
|
<Power className="w-3 h-3" /> On
|
||||||
|
</span>
|
||||||
|
) : (
|
||||||
|
<span className="flex items-center gap-1 text-xs font-medium text-gray-400 bg-gray-100 px-2 py-0.5 rounded-full">
|
||||||
|
<PowerOff className="w-3 h-3" /> Off
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
<p className="text-xs text-gray-400 mb-2">{mt.desc}</p>
|
||||||
|
{config?.model_id ? (
|
||||||
|
<p className="text-xs font-mono text-brand-600 truncate">{config.model_name || config.model_id}</p>
|
||||||
|
) : (
|
||||||
|
<p className="text-xs text-gray-300 italic">No model selected</p>
|
||||||
|
)}
|
||||||
|
</button>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Active Config Editor */}
|
||||||
|
<ConfigEditor
|
||||||
|
key={activeType}
|
||||||
|
moderationType={activeType}
|
||||||
|
config={getConfig(activeType)}
|
||||||
|
onSaved={loadConfigs}
|
||||||
|
/>
|
||||||
|
</AdminShell>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Config Editor for a single moderation type ─────────
|
||||||
|
|
||||||
|
function ConfigEditor({ moderationType, config, onSaved }: {
|
||||||
|
moderationType: string;
|
||||||
|
config?: ModerationConfig;
|
||||||
|
onSaved: () => void;
|
||||||
|
}) {
|
||||||
|
const [modelId, setModelId] = useState(config?.model_id || '');
|
||||||
|
const [modelName, setModelName] = useState(config?.model_name || '');
|
||||||
|
const [systemPrompt, setSystemPrompt] = useState(config?.system_prompt || '');
|
||||||
|
const [enabled, setEnabled] = useState(config?.enabled || false);
|
||||||
|
const [saving, setSaving] = useState(false);
|
||||||
|
|
||||||
|
// Model picker
|
||||||
|
const [showPicker, setShowPicker] = useState(false);
|
||||||
|
const [models, setModels] = useState<ModelInfo[]>([]);
|
||||||
|
const [modelsLoading, setModelsLoading] = useState(false);
|
||||||
|
const [searchTerm, setSearchTerm] = useState('');
|
||||||
|
const [capability, setCapability] = useState('');
|
||||||
|
const searchTimer = useRef<any>(null);
|
||||||
|
|
||||||
|
// Test
|
||||||
|
const [testInput, setTestInput] = useState('');
|
||||||
|
const [testResult, setTestResult] = useState<any>(null);
|
||||||
|
const [testing, setTesting] = useState(false);
|
||||||
|
|
||||||
|
const loadModels = useCallback((search?: string, cap?: string) => {
|
||||||
|
setModelsLoading(true);
|
||||||
|
api.listOpenRouterModels({ search, capability: cap })
|
||||||
|
.then((data) => setModels(data.models || []))
|
||||||
|
.catch(() => {})
|
||||||
|
.finally(() => setModelsLoading(false));
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (showPicker) loadModels(searchTerm || undefined, capability || undefined);
|
||||||
|
}, [showPicker]);
|
||||||
|
|
||||||
|
const onSearchChange = (val: string) => {
|
||||||
|
setSearchTerm(val);
|
||||||
|
if (searchTimer.current) clearTimeout(searchTimer.current);
|
||||||
|
searchTimer.current = setTimeout(() => {
|
||||||
|
loadModels(val || undefined, capability || undefined);
|
||||||
|
}, 400);
|
||||||
|
};
|
||||||
|
|
||||||
|
const onCapabilityChange = (val: string) => {
|
||||||
|
setCapability(val);
|
||||||
|
loadModels(searchTerm || undefined, val || undefined);
|
||||||
|
};
|
||||||
|
|
||||||
|
const selectModel = (m: ModelInfo) => {
|
||||||
|
setModelId(m.id);
|
||||||
|
setModelName(m.name);
|
||||||
|
setShowPicker(false);
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleSave = async () => {
|
||||||
|
setSaving(true);
|
||||||
|
try {
|
||||||
|
await api.setAIModerationConfig({
|
||||||
|
moderation_type: moderationType,
|
||||||
|
model_id: modelId,
|
||||||
|
model_name: modelName,
|
||||||
|
system_prompt: systemPrompt,
|
||||||
|
enabled,
|
||||||
|
});
|
||||||
|
onSaved();
|
||||||
|
} catch (e: any) {
|
||||||
|
alert(e.message);
|
||||||
|
} finally {
|
||||||
|
setSaving(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleTest = async () => {
|
||||||
|
if (!testInput.trim()) return;
|
||||||
|
setTesting(true);
|
||||||
|
setTestResult(null);
|
||||||
|
try {
|
||||||
|
const data = moderationType === 'text'
|
||||||
|
? { moderation_type: moderationType, content: testInput }
|
||||||
|
: { moderation_type: moderationType, image_url: testInput };
|
||||||
|
const res = await api.testAIModeration(data);
|
||||||
|
setTestResult(res.result);
|
||||||
|
} catch (e: any) {
|
||||||
|
setTestResult({ error: e.message });
|
||||||
|
} finally {
|
||||||
|
setTesting(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const isFree = (m: ModelInfo) => m.pricing.prompt === '0' || m.pricing.prompt === '0.0';
|
||||||
|
const isVision = (m: ModelInfo) => {
|
||||||
|
const modality = m.architecture?.modality;
|
||||||
|
return typeof modality === 'string' && modality.includes('image');
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="space-y-4">
|
||||||
|
{/* Model Selection */}
|
||||||
|
<div className="card p-5">
|
||||||
|
<div className="flex items-center justify-between mb-4">
|
||||||
|
<h2 className="font-semibold text-gray-900">
|
||||||
|
{MODERATION_TYPES.find(t => t.key === moderationType)?.label} Configuration
|
||||||
|
</h2>
|
||||||
|
<label className="flex items-center gap-2 cursor-pointer">
|
||||||
|
<span className="text-sm text-gray-500">Enabled</span>
|
||||||
|
<button
|
||||||
|
onClick={() => setEnabled(!enabled)}
|
||||||
|
className={`relative w-10 h-6 rounded-full transition-colors ${enabled ? 'bg-green-500' : 'bg-gray-300'}`}
|
||||||
|
>
|
||||||
|
<span className={`absolute top-0.5 w-5 h-5 bg-white rounded-full shadow transition-transform ${enabled ? 'left-[18px]' : 'left-0.5'}`} />
|
||||||
|
</button>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Selected Model */}
|
||||||
|
<div className="mb-4">
|
||||||
|
<label className="text-sm font-medium text-gray-600 block mb-1">Model</label>
|
||||||
|
<div className="flex gap-2">
|
||||||
|
<div
|
||||||
|
onClick={() => setShowPicker(!showPicker)}
|
||||||
|
className="flex-1 flex items-center justify-between px-3 py-2 border border-warm-300 rounded-lg cursor-pointer hover:bg-warm-50 transition-colors"
|
||||||
|
>
|
||||||
|
{modelId ? (
|
||||||
|
<div>
|
||||||
|
<span className="text-sm font-medium text-gray-900">{modelName || modelId}</span>
|
||||||
|
<span className="text-xs text-gray-400 ml-2 font-mono">{modelId}</span>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<span className="text-sm text-gray-400">Click to select a model...</span>
|
||||||
|
)}
|
||||||
|
<ChevronDown className="w-4 h-4 text-gray-400" />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Model Picker */}
|
||||||
|
{showPicker && (
|
||||||
|
<div className="mb-4 border border-warm-200 rounded-lg overflow-hidden">
|
||||||
|
<div className="p-3 bg-warm-100 flex gap-2">
|
||||||
|
<div className="relative flex-1">
|
||||||
|
<Search className="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-gray-400" />
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
placeholder="Search models..."
|
||||||
|
value={searchTerm}
|
||||||
|
onChange={(e) => onSearchChange(e.target.value)}
|
||||||
|
className="w-full pl-9 pr-3 py-2 text-sm border border-warm-300 rounded-lg focus:outline-none focus:ring-2 focus:ring-brand-500"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<select
|
||||||
|
value={capability}
|
||||||
|
onChange={(e) => onCapabilityChange(e.target.value)}
|
||||||
|
className="text-sm border border-warm-300 rounded-lg px-3 py-2 focus:outline-none focus:ring-2 focus:ring-brand-500"
|
||||||
|
>
|
||||||
|
<option value="">All Models</option>
|
||||||
|
<option value="free">Free Only</option>
|
||||||
|
<option value="vision">Vision Only</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
<div className="max-h-80 overflow-y-auto">
|
||||||
|
{modelsLoading ? (
|
||||||
|
<div className="p-4 text-center text-gray-400 text-sm flex items-center justify-center gap-2">
|
||||||
|
<Loader2 className="w-4 h-4 animate-spin" /> Loading models...
|
||||||
|
</div>
|
||||||
|
) : models.length === 0 ? (
|
||||||
|
<div className="p-4 text-center text-gray-400 text-sm">No models found</div>
|
||||||
|
) : (
|
||||||
|
models.map((m) => (
|
||||||
|
<div
|
||||||
|
key={m.id}
|
||||||
|
onClick={() => selectModel(m)}
|
||||||
|
className={`px-3 py-2.5 border-b border-warm-100 cursor-pointer hover:bg-brand-50 transition-colors ${modelId === m.id ? 'bg-brand-50' : ''}`}
|
||||||
|
>
|
||||||
|
<div className="flex items-center justify-between">
|
||||||
|
<div className="flex items-center gap-2 min-w-0">
|
||||||
|
{modelId === m.id && <Check className="w-4 h-4 text-brand-600 flex-shrink-0" />}
|
||||||
|
<div className="min-w-0">
|
||||||
|
<div className="text-sm font-medium text-gray-900 truncate">{m.name}</div>
|
||||||
|
<div className="text-xs text-gray-400 font-mono truncate">{m.id}</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-center gap-2 flex-shrink-0 ml-2">
|
||||||
|
{isVision(m) && (
|
||||||
|
<span className="text-xs bg-purple-100 text-purple-700 px-1.5 py-0.5 rounded font-medium">Vision</span>
|
||||||
|
)}
|
||||||
|
{isFree(m) ? (
|
||||||
|
<span className="text-xs bg-green-100 text-green-700 px-1.5 py-0.5 rounded font-medium">Free</span>
|
||||||
|
) : (
|
||||||
|
<span className="text-xs text-gray-400">${m.pricing.prompt}/tok</span>
|
||||||
|
)}
|
||||||
|
<span className="text-xs text-gray-300">{(m.context_length / 1000).toFixed(0)}k ctx</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
))
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* System Prompt */}
|
||||||
|
<div className="mb-4">
|
||||||
|
<label className="text-sm font-medium text-gray-600 block mb-1">
|
||||||
|
System Prompt <span className="text-gray-400 font-normal">(leave blank for default)</span>
|
||||||
|
</label>
|
||||||
|
<textarea
|
||||||
|
rows={5}
|
||||||
|
value={systemPrompt}
|
||||||
|
onChange={(e) => setSystemPrompt(e.target.value)}
|
||||||
|
placeholder="Custom system prompt for this moderation type... Leave blank to use the built-in default."
|
||||||
|
className="w-full text-sm border border-warm-300 rounded-lg px-3 py-2 focus:outline-none focus:ring-2 focus:ring-brand-500 font-mono"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Save */}
|
||||||
|
<div className="flex items-center justify-between">
|
||||||
|
<div className="text-xs text-gray-400">
|
||||||
|
{config?.updated_at && `Last updated: ${new Date(config.updated_at).toLocaleString()}`}
|
||||||
|
</div>
|
||||||
|
<button onClick={handleSave} disabled={saving} className="btn-primary text-sm flex items-center gap-1.5">
|
||||||
|
<Sparkles className="w-4 h-4" /> {saving ? 'Saving...' : 'Save Configuration'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Test Panel */}
|
||||||
|
<div className="card p-5">
|
||||||
|
<h3 className="font-semibold text-gray-900 mb-3 flex items-center gap-2">
|
||||||
|
<Play className="w-4 h-4" /> Test Moderation
|
||||||
|
</h3>
|
||||||
|
<p className="text-xs text-gray-400 mb-3">
|
||||||
|
{moderationType === 'text'
|
||||||
|
? 'Enter text content to test moderation'
|
||||||
|
: 'Enter an image URL to test vision moderation'}
|
||||||
|
</p>
|
||||||
|
<div className="flex gap-2 mb-3">
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
value={testInput}
|
||||||
|
onChange={(e) => setTestInput(e.target.value)}
|
||||||
|
placeholder={moderationType === 'text' ? 'Enter test text...' : 'Enter image URL...'}
|
||||||
|
className="flex-1 text-sm border border-warm-300 rounded-lg px-3 py-2 focus:outline-none focus:ring-2 focus:ring-brand-500"
|
||||||
|
/>
|
||||||
|
<button
|
||||||
|
onClick={handleTest}
|
||||||
|
disabled={testing || !modelId || !enabled}
|
||||||
|
className="btn-primary text-sm flex items-center gap-1.5 disabled:opacity-40"
|
||||||
|
>
|
||||||
|
{testing ? <Loader2 className="w-4 h-4 animate-spin" /> : <Play className="w-4 h-4" />}
|
||||||
|
Test
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
{!modelId && <p className="text-xs text-amber-600">Select and save a model first to test</p>}
|
||||||
|
|
||||||
|
{testResult && (
|
||||||
|
<div className={`p-3 rounded-lg text-sm ${testResult.error ? 'bg-red-50 text-red-700' : testResult.flagged ? 'bg-red-50' : 'bg-green-50'}`}>
|
||||||
|
{testResult.error ? (
|
||||||
|
<p>{testResult.error}</p>
|
||||||
|
) : (
|
||||||
|
<div>
|
||||||
|
<div className="flex items-center gap-2 mb-2">
|
||||||
|
<span className={`font-bold ${testResult.flagged ? 'text-red-700' : 'text-green-700'}`}>
|
||||||
|
{testResult.flagged ? 'FLAGGED' : 'CLEAN'}
|
||||||
|
</span>
|
||||||
|
{testResult.reason && <span className="text-gray-600">— {testResult.reason}</span>}
|
||||||
|
</div>
|
||||||
|
<div className="grid grid-cols-3 gap-2">
|
||||||
|
<ScoreBar label="Hate" value={testResult.hate} />
|
||||||
|
<ScoreBar label="Greed" value={testResult.greed} />
|
||||||
|
<ScoreBar label="Delusion" value={testResult.delusion} />
|
||||||
|
</div>
|
||||||
|
{testResult.raw_content && (
|
||||||
|
<details className="mt-2">
|
||||||
|
<summary className="text-xs text-gray-400 cursor-pointer">Raw response</summary>
|
||||||
|
<pre className="mt-1 text-xs bg-white p-2 rounded border border-warm-200 overflow-x-auto whitespace-pre-wrap">{testResult.raw_content}</pre>
|
||||||
|
</details>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function ScoreBar({ label, value }: { label: string; value: number }) {
|
||||||
|
const pct = Math.round((value || 0) * 100);
|
||||||
|
const color = pct > 50 ? 'bg-red-500' : pct > 25 ? 'bg-amber-400' : 'bg-green-400';
|
||||||
|
return (
|
||||||
|
<div>
|
||||||
|
<div className="flex justify-between text-xs mb-0.5">
|
||||||
|
<span className="text-gray-500">{label}</span>
|
||||||
|
<span className="font-mono text-gray-700">{pct}%</span>
|
||||||
|
</div>
|
||||||
|
<div className="h-1.5 bg-gray-200 rounded-full overflow-hidden">
|
||||||
|
<div className={`h-full ${color} rounded-full transition-all`} style={{ width: `${pct}%` }} />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
@ -6,7 +6,7 @@ import { useAuth } from '@/lib/auth';
|
||||||
import { cn } from '@/lib/utils';
|
import { cn } from '@/lib/utils';
|
||||||
import {
|
import {
|
||||||
LayoutDashboard, Users, FileText, Shield, Scale, Flag,
|
LayoutDashboard, Users, FileText, Shield, Scale, Flag,
|
||||||
Settings, Activity, LogOut, ChevronLeft, ChevronRight, Sliders, FolderTree, HardDrive, AtSign,
|
Settings, Activity, LogOut, ChevronLeft, ChevronRight, Sliders, FolderTree, HardDrive, AtSign, Brain,
|
||||||
} from 'lucide-react';
|
} from 'lucide-react';
|
||||||
import { useState } from 'react';
|
import { useState } from 'react';
|
||||||
|
|
||||||
|
|
@ -20,6 +20,7 @@ const navItems = [
|
||||||
{ href: '/algorithm', label: 'Algorithm', icon: Sliders },
|
{ href: '/algorithm', label: 'Algorithm', icon: Sliders },
|
||||||
{ href: '/categories', label: 'Categories', icon: FolderTree },
|
{ href: '/categories', label: 'Categories', icon: FolderTree },
|
||||||
{ href: '/usernames', label: 'Usernames', icon: AtSign },
|
{ href: '/usernames', label: 'Usernames', icon: AtSign },
|
||||||
|
{ href: '/ai-moderation', label: 'AI Moderation', icon: Brain },
|
||||||
{ href: '/storage', label: 'Storage', icon: HardDrive },
|
{ href: '/storage', label: 'Storage', icon: HardDrive },
|
||||||
{ href: '/system', label: 'System Health', icon: Activity },
|
{ href: '/system', label: 'System Health', icon: Activity },
|
||||||
{ href: '/settings', label: 'Settings', icon: Settings },
|
{ href: '/settings', label: 'Settings', icon: Settings },
|
||||||
|
|
|
||||||
|
|
@ -326,6 +326,32 @@ class ApiClient {
|
||||||
body: JSON.stringify({ decision, notes }),
|
body: JSON.stringify({ decision, notes }),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AI Moderation
|
||||||
|
async listOpenRouterModels(params: { capability?: string; search?: string } = {}) {
|
||||||
|
const qs = new URLSearchParams();
|
||||||
|
if (params.capability) qs.set('capability', params.capability);
|
||||||
|
if (params.search) qs.set('search', params.search);
|
||||||
|
return this.request<any>(`/api/v1/admin/ai/models?${qs}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getAIModerationConfigs() {
|
||||||
|
return this.request<any>('/api/v1/admin/ai/config');
|
||||||
|
}
|
||||||
|
|
||||||
|
async setAIModerationConfig(data: { moderation_type: string; model_id: string; model_name: string; system_prompt: string; enabled: boolean }) {
|
||||||
|
return this.request<any>('/api/v1/admin/ai/config', {
|
||||||
|
method: 'PUT',
|
||||||
|
body: JSON.stringify(data),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async testAIModeration(data: { moderation_type: string; content?: string; image_url?: string }) {
|
||||||
|
return this.request<any>('/api/v1/admin/ai/test', {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify(data),
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export const api = new ApiClient();
|
export const api = new ApiClient();
|
||||||
|
|
|
||||||
|
|
@ -121,6 +121,9 @@ func main() {
|
||||||
// Initialize appeal service
|
// Initialize appeal service
|
||||||
appealService := services.NewAppealService(dbPool)
|
appealService := services.NewAppealService(dbPool)
|
||||||
|
|
||||||
|
// Initialize OpenRouter service
|
||||||
|
openRouterService := services.NewOpenRouterService(dbPool, cfg.OpenRouterAPIKey)
|
||||||
|
|
||||||
// Initialize content filter (hard blocklist + strike system)
|
// Initialize content filter (hard blocklist + strike system)
|
||||||
contentFilter := services.NewContentFilter(dbPool)
|
contentFilter := services.NewContentFilter(dbPool)
|
||||||
|
|
||||||
|
|
@ -155,7 +158,7 @@ func main() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
adminHandler := handlers.NewAdminHandler(dbPool, moderationService, appealService, emailService, cfg.JWTSecret, cfg.TurnstileSecretKey, s3Client, cfg.R2MediaBucket, cfg.R2VideoBucket, cfg.R2ImgDomain, cfg.R2VidDomain)
|
adminHandler := handlers.NewAdminHandler(dbPool, moderationService, appealService, emailService, openRouterService, cfg.JWTSecret, cfg.TurnstileSecretKey, s3Client, cfg.R2MediaBucket, cfg.R2VideoBucket, cfg.R2ImgDomain, cfg.R2VidDomain)
|
||||||
|
|
||||||
mediaHandler := handlers.NewMediaHandler(
|
mediaHandler := handlers.NewMediaHandler(
|
||||||
s3Client,
|
s3Client,
|
||||||
|
|
@ -443,6 +446,12 @@ func main() {
|
||||||
// Username Claim Requests
|
// Username Claim Requests
|
||||||
admin.GET("/usernames/claims", adminHandler.ListClaimRequests)
|
admin.GET("/usernames/claims", adminHandler.ListClaimRequests)
|
||||||
admin.PATCH("/usernames/claims/:id", adminHandler.ReviewClaimRequest)
|
admin.PATCH("/usernames/claims/:id", adminHandler.ReviewClaimRequest)
|
||||||
|
|
||||||
|
// AI Moderation Config
|
||||||
|
admin.GET("/ai/models", adminHandler.ListOpenRouterModels)
|
||||||
|
admin.GET("/ai/config", adminHandler.GetAIModerationConfigs)
|
||||||
|
admin.PUT("/ai/config", adminHandler.SetAIModerationConfig)
|
||||||
|
admin.POST("/ai/test", adminHandler.TestAIModeration)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Public claim request endpoint (no auth)
|
// Public claim request endpoint (no auth)
|
||||||
|
|
|
||||||
|
|
@ -38,6 +38,7 @@ type Config struct {
|
||||||
TurnstileSecretKey string
|
TurnstileSecretKey string
|
||||||
APIBaseURL string
|
APIBaseURL string
|
||||||
AppBaseURL string
|
AppBaseURL string
|
||||||
|
OpenRouterAPIKey string
|
||||||
}
|
}
|
||||||
|
|
||||||
func LoadConfig() *Config {
|
func LoadConfig() *Config {
|
||||||
|
|
@ -82,6 +83,7 @@ func LoadConfig() *Config {
|
||||||
TurnstileSecretKey: getEnv("TURNSTILE_SECRET", ""),
|
TurnstileSecretKey: getEnv("TURNSTILE_SECRET", ""),
|
||||||
APIBaseURL: getEnv("API_BASE_URL", "https://api.sojorn.net"),
|
APIBaseURL: getEnv("API_BASE_URL", "https://api.sojorn.net"),
|
||||||
AppBaseURL: getEnv("APP_BASE_URL", "https://sojorn.net"),
|
AppBaseURL: getEnv("APP_BASE_URL", "https://sojorn.net"),
|
||||||
|
OpenRouterAPIKey: getEnv("OPENROUTER_API", ""),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,7 @@ type AdminHandler struct {
|
||||||
moderationService *services.ModerationService
|
moderationService *services.ModerationService
|
||||||
appealService *services.AppealService
|
appealService *services.AppealService
|
||||||
emailService *services.EmailService
|
emailService *services.EmailService
|
||||||
|
openRouterService *services.OpenRouterService
|
||||||
jwtSecret string
|
jwtSecret string
|
||||||
turnstileSecret string
|
turnstileSecret string
|
||||||
s3Client *s3.Client
|
s3Client *s3.Client
|
||||||
|
|
@ -34,12 +35,13 @@ type AdminHandler struct {
|
||||||
vidDomain string
|
vidDomain string
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewAdminHandler(pool *pgxpool.Pool, moderationService *services.ModerationService, appealService *services.AppealService, emailService *services.EmailService, jwtSecret string, turnstileSecret string, s3Client *s3.Client, mediaBucket string, videoBucket string, imgDomain string, vidDomain string) *AdminHandler {
|
func NewAdminHandler(pool *pgxpool.Pool, moderationService *services.ModerationService, appealService *services.AppealService, emailService *services.EmailService, openRouterService *services.OpenRouterService, jwtSecret string, turnstileSecret string, s3Client *s3.Client, mediaBucket string, videoBucket string, imgDomain string, vidDomain string) *AdminHandler {
|
||||||
return &AdminHandler{
|
return &AdminHandler{
|
||||||
pool: pool,
|
pool: pool,
|
||||||
moderationService: moderationService,
|
moderationService: moderationService,
|
||||||
appealService: appealService,
|
appealService: appealService,
|
||||||
emailService: emailService,
|
emailService: emailService,
|
||||||
|
openRouterService: openRouterService,
|
||||||
jwtSecret: jwtSecret,
|
jwtSecret: jwtSecret,
|
||||||
turnstileSecret: turnstileSecret,
|
turnstileSecret: turnstileSecret,
|
||||||
s3Client: s3Client,
|
s3Client: s3Client,
|
||||||
|
|
@ -2304,3 +2306,152 @@ func (h *AdminHandler) SubmitClaimRequest(c *gin.Context) {
|
||||||
|
|
||||||
c.JSON(http.StatusOK, gin.H{"message": "Your claim request has been submitted and will be reviewed by our team."})
|
c.JSON(http.StatusOK, gin.H{"message": "Your claim request has been submitted and will be reviewed by our team."})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ──────────────────────────────────────────────
|
||||||
|
// AI Moderation Config
|
||||||
|
// ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
func (h *AdminHandler) ListOpenRouterModels(c *gin.Context) {
|
||||||
|
if h.openRouterService == nil {
|
||||||
|
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "OpenRouter not configured"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
models, err := h.openRouterService.ListModels(c.Request.Context())
|
||||||
|
if err != nil {
|
||||||
|
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to fetch models: %v", err)})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Optional filter by capability
|
||||||
|
capability := c.Query("capability") // "text", "image", "vision", "free"
|
||||||
|
if capability != "" {
|
||||||
|
var filtered []services.OpenRouterModel
|
||||||
|
for _, m := range models {
|
||||||
|
switch capability {
|
||||||
|
case "free":
|
||||||
|
if m.Pricing.Prompt == "0" || m.Pricing.Prompt == "0.0" {
|
||||||
|
filtered = append(filtered, m)
|
||||||
|
}
|
||||||
|
case "vision", "image":
|
||||||
|
if arch, ok := m.Architecture["modality"].(string); ok {
|
||||||
|
if strings.Contains(arch, "image") {
|
||||||
|
filtered = append(filtered, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
filtered = append(filtered, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
models = filtered
|
||||||
|
}
|
||||||
|
|
||||||
|
// Search filter
|
||||||
|
search := strings.ToLower(c.Query("search"))
|
||||||
|
if search != "" {
|
||||||
|
var filtered []services.OpenRouterModel
|
||||||
|
for _, m := range models {
|
||||||
|
if strings.Contains(strings.ToLower(m.ID), search) || strings.Contains(strings.ToLower(m.Name), search) {
|
||||||
|
filtered = append(filtered, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
models = filtered
|
||||||
|
}
|
||||||
|
|
||||||
|
c.JSON(http.StatusOK, gin.H{"models": models, "total": len(models)})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *AdminHandler) GetAIModerationConfigs(c *gin.Context) {
|
||||||
|
if h.openRouterService == nil {
|
||||||
|
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "OpenRouter not configured"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
configs, err := h.openRouterService.GetModerationConfigs(c.Request.Context())
|
||||||
|
if err != nil {
|
||||||
|
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to fetch configs: %v", err)})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
c.JSON(http.StatusOK, gin.H{"configs": configs})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *AdminHandler) SetAIModerationConfig(c *gin.Context) {
|
||||||
|
if h.openRouterService == nil {
|
||||||
|
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "OpenRouter not configured"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var req struct {
|
||||||
|
ModerationType string `json:"moderation_type" binding:"required"`
|
||||||
|
ModelID string `json:"model_id"`
|
||||||
|
ModelName string `json:"model_name"`
|
||||||
|
SystemPrompt string `json:"system_prompt"`
|
||||||
|
Enabled bool `json:"enabled"`
|
||||||
|
}
|
||||||
|
if err := c.ShouldBindJSON(&req); err != nil {
|
||||||
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if req.ModerationType != "text" && req.ModerationType != "image" && req.ModerationType != "video" {
|
||||||
|
c.JSON(http.StatusBadRequest, gin.H{"error": "moderation_type must be text, image, or video"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
adminID := c.GetString("user_id")
|
||||||
|
err := h.openRouterService.SetModerationConfig(c.Request.Context(), req.ModerationType, req.ModelID, req.ModelName, req.SystemPrompt, req.Enabled, adminID)
|
||||||
|
if err != nil {
|
||||||
|
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to save config: %v", err)})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Audit log
|
||||||
|
h.pool.Exec(c.Request.Context(), `
|
||||||
|
INSERT INTO audit_log (admin_id, action, target_type, details)
|
||||||
|
VALUES ($1, 'update_ai_moderation', 'ai_config', $2)
|
||||||
|
`, adminID, fmt.Sprintf("Set %s moderation model to %s (enabled=%v)", req.ModerationType, req.ModelID, req.Enabled))
|
||||||
|
|
||||||
|
c.JSON(http.StatusOK, gin.H{"message": "Configuration updated"})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *AdminHandler) TestAIModeration(c *gin.Context) {
|
||||||
|
if h.openRouterService == nil {
|
||||||
|
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "OpenRouter not configured"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var req struct {
|
||||||
|
ModerationType string `json:"moderation_type" binding:"required"`
|
||||||
|
Content string `json:"content"`
|
||||||
|
ImageURL string `json:"image_url"`
|
||||||
|
}
|
||||||
|
if err := c.ShouldBindJSON(&req); err != nil {
|
||||||
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := c.Request.Context()
|
||||||
|
var result *services.ModerationResult
|
||||||
|
var err error
|
||||||
|
|
||||||
|
switch req.ModerationType {
|
||||||
|
case "text":
|
||||||
|
result, err = h.openRouterService.ModerateText(ctx, req.Content)
|
||||||
|
case "image":
|
||||||
|
result, err = h.openRouterService.ModerateImage(ctx, req.ImageURL)
|
||||||
|
case "video":
|
||||||
|
urls := strings.Split(req.ImageURL, ",")
|
||||||
|
result, err = h.openRouterService.ModerateVideo(ctx, urls)
|
||||||
|
default:
|
||||||
|
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid moderation_type"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
c.JSON(http.StatusOK, gin.H{"result": result})
|
||||||
|
}
|
||||||
|
|
|
||||||
366
go-backend/internal/services/openrouter_service.go
Normal file
366
go-backend/internal/services/openrouter_service.go
Normal file
|
|
@ -0,0 +1,366 @@
|
||||||
|
package services
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
)
|
||||||
|
|
||||||
|
// OpenRouterService handles interactions with the OpenRouter API
|
||||||
|
type OpenRouterService struct {
|
||||||
|
pool *pgxpool.Pool
|
||||||
|
httpClient *http.Client
|
||||||
|
apiKey string
|
||||||
|
|
||||||
|
// Cached model list
|
||||||
|
modelCache []OpenRouterModel
|
||||||
|
modelCacheMu sync.RWMutex
|
||||||
|
modelCacheTime time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenRouterModel represents a model available on OpenRouter
|
||||||
|
type OpenRouterModel struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Description string `json:"description,omitempty"`
|
||||||
|
Pricing OpenRouterPricing `json:"pricing"`
|
||||||
|
ContextLength int `json:"context_length"`
|
||||||
|
Architecture map[string]any `json:"architecture,omitempty"`
|
||||||
|
TopProvider map[string]any `json:"top_provider,omitempty"`
|
||||||
|
PerRequestLimits map[string]any `json:"per_request_limits,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type OpenRouterPricing struct {
|
||||||
|
Prompt string `json:"prompt"`
|
||||||
|
Completion string `json:"completion"`
|
||||||
|
Image string `json:"image,omitempty"`
|
||||||
|
Request string `json:"request,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ModerationConfigEntry represents a row in ai_moderation_config
|
||||||
|
type ModerationConfigEntry struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
ModerationType string `json:"moderation_type"`
|
||||||
|
ModelID string `json:"model_id"`
|
||||||
|
ModelName string `json:"model_name"`
|
||||||
|
SystemPrompt string `json:"system_prompt"`
|
||||||
|
Enabled bool `json:"enabled"`
|
||||||
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
|
UpdatedBy *string `json:"updated_by,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenRouterChatMessage represents a message in a chat completion request
|
||||||
|
type OpenRouterChatMessage struct {
|
||||||
|
Role string `json:"role"`
|
||||||
|
Content any `json:"content"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenRouterChatRequest represents a chat completion request
|
||||||
|
type OpenRouterChatRequest struct {
|
||||||
|
Model string `json:"model"`
|
||||||
|
Messages []OpenRouterChatMessage `json:"messages"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenRouterChatResponse represents a chat completion response
|
||||||
|
type OpenRouterChatResponse struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Choices []struct {
|
||||||
|
Message struct {
|
||||||
|
Role string `json:"role"`
|
||||||
|
Content string `json:"content"`
|
||||||
|
} `json:"message"`
|
||||||
|
FinishReason string `json:"finish_reason"`
|
||||||
|
} `json:"choices"`
|
||||||
|
Usage struct {
|
||||||
|
PromptTokens int `json:"prompt_tokens"`
|
||||||
|
CompletionTokens int `json:"completion_tokens"`
|
||||||
|
TotalTokens int `json:"total_tokens"`
|
||||||
|
} `json:"usage"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewOpenRouterService(pool *pgxpool.Pool, apiKey string) *OpenRouterService {
|
||||||
|
return &OpenRouterService{
|
||||||
|
pool: pool,
|
||||||
|
apiKey: apiKey,
|
||||||
|
httpClient: &http.Client{
|
||||||
|
Timeout: 60 * time.Second,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListModels fetches available models from OpenRouter, with 1-hour cache
|
||||||
|
func (s *OpenRouterService) ListModels(ctx context.Context) ([]OpenRouterModel, error) {
|
||||||
|
s.modelCacheMu.RLock()
|
||||||
|
if len(s.modelCache) > 0 && time.Since(s.modelCacheTime) < time.Hour {
|
||||||
|
cached := s.modelCache
|
||||||
|
s.modelCacheMu.RUnlock()
|
||||||
|
return cached, nil
|
||||||
|
}
|
||||||
|
s.modelCacheMu.RUnlock()
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, "GET", "https://openrouter.ai/api/v1/models", nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||||
|
}
|
||||||
|
if s.apiKey != "" {
|
||||||
|
req.Header.Set("Authorization", "Bearer "+s.apiKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := s.httpClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to fetch models: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
return nil, fmt.Errorf("OpenRouter API error %d: %s", resp.StatusCode, string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
var result struct {
|
||||||
|
Data []OpenRouterModel `json:"data"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to decode models: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.modelCacheMu.Lock()
|
||||||
|
s.modelCache = result.Data
|
||||||
|
s.modelCacheTime = time.Now()
|
||||||
|
s.modelCacheMu.Unlock()
|
||||||
|
|
||||||
|
return result.Data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetModerationConfigs returns all moderation type configurations
|
||||||
|
func (s *OpenRouterService) GetModerationConfigs(ctx context.Context) ([]ModerationConfigEntry, error) {
|
||||||
|
rows, err := s.pool.Query(ctx, `
|
||||||
|
SELECT id, moderation_type, model_id, model_name, system_prompt, enabled, updated_at, updated_by
|
||||||
|
FROM ai_moderation_config
|
||||||
|
ORDER BY moderation_type
|
||||||
|
`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to query configs: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
var configs []ModerationConfigEntry
|
||||||
|
for rows.Next() {
|
||||||
|
var c ModerationConfigEntry
|
||||||
|
if err := rows.Scan(&c.ID, &c.ModerationType, &c.ModelID, &c.ModelName, &c.SystemPrompt, &c.Enabled, &c.UpdatedAt, &c.UpdatedBy); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
configs = append(configs, c)
|
||||||
|
}
|
||||||
|
return configs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetModerationConfig returns config for a specific moderation type
|
||||||
|
func (s *OpenRouterService) GetModerationConfig(ctx context.Context, moderationType string) (*ModerationConfigEntry, error) {
|
||||||
|
var c ModerationConfigEntry
|
||||||
|
err := s.pool.QueryRow(ctx, `
|
||||||
|
SELECT id, moderation_type, model_id, model_name, system_prompt, enabled, updated_at, updated_by
|
||||||
|
FROM ai_moderation_config WHERE moderation_type = $1
|
||||||
|
`, moderationType).Scan(&c.ID, &c.ModerationType, &c.ModelID, &c.ModelName, &c.SystemPrompt, &c.Enabled, &c.UpdatedAt, &c.UpdatedBy)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &c, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetModerationConfig upserts a moderation config
|
||||||
|
func (s *OpenRouterService) SetModerationConfig(ctx context.Context, moderationType, modelID, modelName, systemPrompt string, enabled bool, updatedBy string) error {
|
||||||
|
_, err := s.pool.Exec(ctx, `
|
||||||
|
INSERT INTO ai_moderation_config (moderation_type, model_id, model_name, system_prompt, enabled, updated_by, updated_at)
|
||||||
|
VALUES ($1, $2, $3, $4, $5, $6, NOW())
|
||||||
|
ON CONFLICT (moderation_type)
|
||||||
|
DO UPDATE SET model_id = $2, model_name = $3, system_prompt = $4, enabled = $5, updated_by = $6, updated_at = NOW()
|
||||||
|
`, moderationType, modelID, modelName, systemPrompt, enabled, updatedBy)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// ModerateText sends text content to the configured model for moderation
|
||||||
|
func (s *OpenRouterService) ModerateText(ctx context.Context, content string) (*ModerationResult, error) {
|
||||||
|
config, err := s.GetModerationConfig(ctx, "text")
|
||||||
|
if err != nil || !config.Enabled || config.ModelID == "" {
|
||||||
|
return nil, fmt.Errorf("text moderation not configured")
|
||||||
|
}
|
||||||
|
return s.callModel(ctx, config.ModelID, config.SystemPrompt, content, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ModerateImage sends an image URL to a vision model for moderation
|
||||||
|
func (s *OpenRouterService) ModerateImage(ctx context.Context, imageURL string) (*ModerationResult, error) {
|
||||||
|
config, err := s.GetModerationConfig(ctx, "image")
|
||||||
|
if err != nil || !config.Enabled || config.ModelID == "" {
|
||||||
|
return nil, fmt.Errorf("image moderation not configured")
|
||||||
|
}
|
||||||
|
return s.callModel(ctx, config.ModelID, config.SystemPrompt, "", []string{imageURL})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ModerateVideo sends video frame URLs to a vision model for moderation
|
||||||
|
func (s *OpenRouterService) ModerateVideo(ctx context.Context, frameURLs []string) (*ModerationResult, error) {
|
||||||
|
config, err := s.GetModerationConfig(ctx, "video")
|
||||||
|
if err != nil || !config.Enabled || config.ModelID == "" {
|
||||||
|
return nil, fmt.Errorf("video moderation not configured")
|
||||||
|
}
|
||||||
|
return s.callModel(ctx, config.ModelID, config.SystemPrompt, "These are 3 frames extracted from a short video. Analyze all frames for policy violations.", frameURLs)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ModerationResult is the parsed response from OpenRouter moderation
|
||||||
|
type ModerationResult struct {
|
||||||
|
Flagged bool `json:"flagged"`
|
||||||
|
Reason string `json:"reason"`
|
||||||
|
Hate float64 `json:"hate"`
|
||||||
|
Greed float64 `json:"greed"`
|
||||||
|
Delusion float64 `json:"delusion"`
|
||||||
|
RawContent string `json:"raw_content"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// callModel sends a chat completion request to OpenRouter
|
||||||
|
func (s *OpenRouterService) callModel(ctx context.Context, modelID, systemPrompt, textContent string, imageURLs []string) (*ModerationResult, error) {
|
||||||
|
if s.apiKey == "" {
|
||||||
|
return nil, fmt.Errorf("OpenRouter API key not configured")
|
||||||
|
}
|
||||||
|
|
||||||
|
messages := []OpenRouterChatMessage{}
|
||||||
|
|
||||||
|
// System prompt
|
||||||
|
if systemPrompt == "" {
|
||||||
|
systemPrompt = defaultModerationSystemPrompt
|
||||||
|
}
|
||||||
|
messages = append(messages, OpenRouterChatMessage{Role: "system", Content: systemPrompt})
|
||||||
|
|
||||||
|
// User message — text only or multimodal (text + images)
|
||||||
|
if len(imageURLs) > 0 {
|
||||||
|
// Multimodal content array
|
||||||
|
parts := []map[string]any{}
|
||||||
|
if textContent != "" {
|
||||||
|
parts = append(parts, map[string]any{"type": "text", "text": textContent})
|
||||||
|
}
|
||||||
|
for _, url := range imageURLs {
|
||||||
|
parts = append(parts, map[string]any{
|
||||||
|
"type": "image_url",
|
||||||
|
"image_url": map[string]string{"url": url},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
messages = append(messages, OpenRouterChatMessage{Role: "user", Content: parts})
|
||||||
|
} else {
|
||||||
|
messages = append(messages, OpenRouterChatMessage{Role: "user", Content: textContent})
|
||||||
|
}
|
||||||
|
|
||||||
|
reqBody := OpenRouterChatRequest{
|
||||||
|
Model: modelID,
|
||||||
|
Messages: messages,
|
||||||
|
}
|
||||||
|
|
||||||
|
jsonBody, err := json.Marshal(reqBody)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to marshal request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, "POST", "https://openrouter.ai/api/v1/chat/completions", bytes.NewBuffer(jsonBody))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("Authorization", "Bearer "+s.apiKey)
|
||||||
|
req.Header.Set("HTTP-Referer", "https://sojorn.net")
|
||||||
|
req.Header.Set("X-Title", "Sojorn Moderation")
|
||||||
|
|
||||||
|
resp, err := s.httpClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("OpenRouter request failed: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
return nil, fmt.Errorf("OpenRouter API error %d: %s", resp.StatusCode, string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
var chatResp OpenRouterChatResponse
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&chatResp); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to decode response: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(chatResp.Choices) == 0 {
|
||||||
|
return nil, fmt.Errorf("no response from model")
|
||||||
|
}
|
||||||
|
|
||||||
|
raw := chatResp.Choices[0].Message.Content
|
||||||
|
return parseModerationResponse(raw), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseModerationResponse tries to extract structured moderation data from model output
|
||||||
|
func parseModerationResponse(raw string) *ModerationResult {
|
||||||
|
result := &ModerationResult{RawContent: raw}
|
||||||
|
|
||||||
|
// Try to parse JSON from the response
|
||||||
|
// Models may wrap JSON in markdown code blocks
|
||||||
|
cleaned := raw
|
||||||
|
if idx := strings.Index(cleaned, "```json"); idx >= 0 {
|
||||||
|
cleaned = cleaned[idx+7:]
|
||||||
|
if end := strings.Index(cleaned, "```"); end >= 0 {
|
||||||
|
cleaned = cleaned[:end]
|
||||||
|
}
|
||||||
|
} else if idx := strings.Index(cleaned, "```"); idx >= 0 {
|
||||||
|
cleaned = cleaned[idx+3:]
|
||||||
|
if end := strings.Index(cleaned, "```"); end >= 0 {
|
||||||
|
cleaned = cleaned[:end]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cleaned = strings.TrimSpace(cleaned)
|
||||||
|
|
||||||
|
var parsed struct {
|
||||||
|
Flagged bool `json:"flagged"`
|
||||||
|
Reason string `json:"reason"`
|
||||||
|
Hate float64 `json:"hate"`
|
||||||
|
Greed float64 `json:"greed"`
|
||||||
|
Delusion float64 `json:"delusion"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(cleaned), &parsed); err == nil {
|
||||||
|
result.Flagged = parsed.Flagged
|
||||||
|
result.Reason = parsed.Reason
|
||||||
|
result.Hate = parsed.Hate
|
||||||
|
result.Greed = parsed.Greed
|
||||||
|
result.Delusion = parsed.Delusion
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: check for keywords in raw text
|
||||||
|
lower := strings.ToLower(raw)
|
||||||
|
if strings.Contains(lower, "violation") || strings.Contains(lower, "inappropriate") || strings.Contains(lower, "flagged") {
|
||||||
|
result.Flagged = true
|
||||||
|
result.Reason = "Content flagged by AI moderation"
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
const defaultModerationSystemPrompt = `You are a content moderation AI for Sojorn, a social media platform.
|
||||||
|
Analyze the provided content for policy violations.
|
||||||
|
|
||||||
|
Respond ONLY with a JSON object in this exact format:
|
||||||
|
{
|
||||||
|
"flagged": true/false,
|
||||||
|
"reason": "brief reason if flagged, empty string if not",
|
||||||
|
"hate": 0.0-1.0,
|
||||||
|
"greed": 0.0-1.0,
|
||||||
|
"delusion": 0.0-1.0
|
||||||
|
}
|
||||||
|
|
||||||
|
Scoring guide (Three Poisons framework):
|
||||||
|
- hate: harassment, threats, violence, sexual content, hate speech, discrimination
|
||||||
|
- greed: spam, scams, crypto schemes, misleading promotions, get-rich-quick
|
||||||
|
- delusion: misinformation, self-harm content, conspiracy theories, dangerous medical advice
|
||||||
|
|
||||||
|
Score 0.0 = no concern, 1.0 = extreme violation. Flag if any score > 0.5.
|
||||||
|
Only respond with the JSON, no other text.`
|
||||||
|
|
@ -12,7 +12,7 @@ import '../../../services/image_upload_service.dart';
|
||||||
import '../../../theme/app_theme.dart';
|
import '../../../theme/app_theme.dart';
|
||||||
import '../../compose/video_editor_screen.dart';
|
import '../../compose/video_editor_screen.dart';
|
||||||
|
|
||||||
/// Quip editor for videos with 15-second maximum duration.
|
/// Quip editor for videos with 10-second maximum duration.
|
||||||
/// Uses the standardized sojornVideoEditor and uploads via ImageUploadService.
|
/// Uses the standardized sojornVideoEditor and uploads via ImageUploadService.
|
||||||
class QuipEditorScreen extends ConsumerStatefulWidget {
|
class QuipEditorScreen extends ConsumerStatefulWidget {
|
||||||
final File videoFile;
|
final File videoFile;
|
||||||
|
|
@ -31,7 +31,7 @@ class QuipEditorScreen extends ConsumerStatefulWidget {
|
||||||
}
|
}
|
||||||
|
|
||||||
class _QuipEditorScreenState extends ConsumerState<QuipEditorScreen> {
|
class _QuipEditorScreenState extends ConsumerState<QuipEditorScreen> {
|
||||||
static const Duration _maxDuration = Duration(seconds: 15);
|
static const Duration _maxDuration = Duration(seconds: 10);
|
||||||
final TextEditingController _captionController = TextEditingController();
|
final TextEditingController _captionController = TextEditingController();
|
||||||
final ImageUploadService _uploadService = ImageUploadService();
|
final ImageUploadService _uploadService = ImageUploadService();
|
||||||
|
|
||||||
|
|
@ -167,7 +167,7 @@ class _QuipEditorScreenState extends ConsumerState<QuipEditorScreen> {
|
||||||
padding: const EdgeInsets.only(right: 16),
|
padding: const EdgeInsets.only(right: 16),
|
||||||
child: Center(
|
child: Center(
|
||||||
child: Text(
|
child: Text(
|
||||||
'Trim required (max 15s)',
|
'Trim required (max 10s)',
|
||||||
style: TextStyle(
|
style: TextStyle(
|
||||||
color: AppTheme.error,
|
color: AppTheme.error,
|
||||||
fontSize: 12,
|
fontSize: 12,
|
||||||
|
|
@ -211,7 +211,7 @@ class _QuipEditorScreenState extends ConsumerState<QuipEditorScreen> {
|
||||||
if (!videoDurationOk) ...[
|
if (!videoDurationOk) ...[
|
||||||
const SizedBox(height: 8),
|
const SizedBox(height: 8),
|
||||||
Text(
|
Text(
|
||||||
'Video exceeds 15s - trim required',
|
'Video exceeds 10s - trim required',
|
||||||
style: TextStyle(
|
style: TextStyle(
|
||||||
color: AppTheme.error,
|
color: AppTheme.error,
|
||||||
fontSize: 12,
|
fontSize: 12,
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ class QuipRecorderScreen extends StatefulWidget {
|
||||||
class _QuipRecorderScreenState extends State<QuipRecorderScreen>
|
class _QuipRecorderScreenState extends State<QuipRecorderScreen>
|
||||||
with WidgetsBindingObserver {
|
with WidgetsBindingObserver {
|
||||||
// Config
|
// Config
|
||||||
static const Duration _maxDuration = Duration(seconds: 60);
|
static const Duration _maxDuration = Duration(seconds: 10);
|
||||||
|
|
||||||
// Camera State
|
// Camera State
|
||||||
CameraController? _cameraController;
|
CameraController? _cameraController;
|
||||||
|
|
@ -222,7 +222,7 @@ class _QuipRecorderScreenState extends State<QuipRecorderScreen>
|
||||||
|
|
||||||
Future<void> _pickFromGallery() async {
|
Future<void> _pickFromGallery() async {
|
||||||
final picker = ImagePicker();
|
final picker = ImagePicker();
|
||||||
final video = await picker.pickVideo(source: ImageSource.gallery, maxDuration: _maxDuration);
|
final video = await picker.pickVideo(source: ImageSource.gallery, maxDuration: const Duration(seconds: 10));
|
||||||
if (video != null) {
|
if (video != null) {
|
||||||
if (mounted) {
|
if (mounted) {
|
||||||
Navigator.push(
|
Navigator.push(
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue