patrick/sojorn
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

feat: Audio overlay system — AudioLibraryScreen, Funkwhale proxy, ffmpeg audio mix

Browse source 
Go:
- GET /audio/library?q= — Funkwhale tracks proxy (503 until FUNKWHALE_BASE set)
- GET /audio/library/:trackId/listen — audio stream proxy
- FUNKWHALE_BASE config key added (env var)

Flutter:
- AudioLibraryScreen: Device tab (file_picker) + Library tab (Funkwhale)
- VideoStitchingService.stitchVideos(): audioOverlayPath + audioVolume params
  — second FFmpeg pass: amix with configurable volume, falls back if mix fails
- EnhancedQuipRecorderScreen: music button, audio chip + volume slider, wired to stitcher

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Patrick Britton 2026-02-17 16:00:55 -06:00
parent 6e2de2cd9d
commit 135bb7f08d
6 changed files with 462 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
go-backend/cmd/api/main.go
View file

@ -224,6 +224,9 @@ func main() {
repostHandler := handlers.NewRepostHandler(dbPool)
profileLayoutHandler := handlers.NewProfileLayoutHandler(dbPool)
// Audio library proxy (Funkwhale — gracefully returns 503 until FUNKWHALE_BASE is set)
audioHandler := handlers.NewAudioHandler(cfg.FunkwhaleBase)
r.GET("/ws", wsHandler.ServeWS)
r.GET("/health", func(c *gin.Context) {
@ -583,6 +586,10 @@ func main() {
authorized.GET("/profile/layout", profileLayoutHandler.GetProfileLayout)
authorized.PUT("/profile/layout", profileLayoutHandler.SaveProfileLayout)
// Audio library (Funkwhale proxy — returns 503 until FUNKWHALE_BASE is set in env)
authorized.GET("/audio/library", audioHandler.SearchAudioLibrary)
authorized.GET("/audio/library/:trackId/listen", audioHandler.GetAudioTrackListen)
}
}

2
go-backend/internal/config/config.go
View file

@ -43,6 +43,7 @@ type Config struct {
AzureOpenAIAPIKey string
AzureOpenAIEndpoint string
AzureOpenAIAPIVersion string
FunkwhaleBase string // e.g. "http://localhost:5001" — empty means not yet deployed
}
func LoadConfig() *Config {
@ -92,6 +93,7 @@ func LoadConfig() *Config {
AzureOpenAIAPIKey: getEnv("AZURE_OPENAI_API_KEY", ""),
AzureOpenAIEndpoint: getEnv("AZURE_OPENAI_ENDPOINT", ""),
AzureOpenAIAPIVersion: getEnv("AZURE_OPENAI_API_VERSION", "2024-02-15-preview"),
FunkwhaleBase: getEnv("FUNKWHALE_BASE", ""),
}
}

78
go-backend/internal/handlers/audio_handler.go Normal file
View file

@ -0,0 +1,78 @@
package handlers
import (
"fmt"
"io"
"net/http"
"net/url"
"github.com/gin-gonic/gin"
)
// AudioHandler proxies Funkwhale audio library requests so the Flutter app
// doesn't need CORS credentials or direct Funkwhale access.
type AudioHandler struct {
funkwhaleBase string // e.g. "http://localhost:5001" — empty = not yet deployed
}
func NewAudioHandler(funkwhaleBase string) *AudioHandler {
return &AudioHandler{funkwhaleBase: funkwhaleBase}
}
// SearchAudioLibrary proxies GET /audio/library?q=&page= to Funkwhale /api/v1/tracks/
func (h *AudioHandler) SearchAudioLibrary(c *gin.Context) {
if h.funkwhaleBase == "" {
c.JSON(http.StatusServiceUnavailable, gin.H{
"error": "audio library not yet configured — Funkwhale deployment pending",
"tracks": []any{},
"count": 0,
})
return
}
q := url.QueryEscape(c.DefaultQuery("q", ""))
page := url.QueryEscape(c.DefaultQuery("page", "1"))
target := fmt.Sprintf("%s/api/v1/tracks/?q=%s&page=%s&playable=true", h.funkwhaleBase, q, page)
resp, err := http.Get(target) //nolint:gosec
if err != nil {
c.JSON(http.StatusBadGateway, gin.H{"error": "audio library unavailable"})
return
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
c.Data(resp.StatusCode, "application/json", body)
}
// GetAudioTrackListen proxies the audio stream for a track.
// Flutter uses this URL in ffmpeg_kit as the audio input.
func (h *AudioHandler) GetAudioTrackListen(c *gin.Context) {
if h.funkwhaleBase == "" {
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "audio library not yet configured"})
return
}
trackID := c.Param("trackId")
target := fmt.Sprintf("%s/api/v1/listen/%s/", h.funkwhaleBase, url.PathEscape(trackID))
req, err := http.NewRequestWithContext(c.Request.Context(), http.MethodGet, target, nil)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create request"})
return
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
c.JSON(http.StatusBadGateway, gin.H{"error": "audio stream unavailable"})
return
}
defer resp.Body.Close()
contentType := resp.Header.Get("Content-Type")
if contentType == "" {
contentType = "audio/mpeg"
}
c.DataFromReader(resp.StatusCode, resp.ContentLength, contentType, resp.Body, nil)
}

280
sojorn_app/lib/screens/audio/audio_library_screen.dart Normal file
View file

@ -0,0 +1,280 @@
import 'dart:io';
import 'package:file_picker/file_picker.dart';
import 'package:flutter/material.dart';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:video_player/video_player.dart';
import '../../config/api_config.dart';
import '../../providers/api_provider.dart';
/// Result returned when the user picks an audio track.
class AudioTrack {
final String path; // local file path OR network URL (feed directly to ffmpeg)
final String title;
const AudioTrack({required this.path, required this.title});
}
/// Two-tab screen for picking background audio.
///
/// Tab 1 (Device): opens the file picker for local audio files.
/// Tab 2 (Library): browses the Funkwhale library via the Go proxy.
///
/// Navigator.push returns an [AudioTrack] when the user picks a track,
/// or null if they cancelled.
class AudioLibraryScreen extends ConsumerStatefulWidget {
const AudioLibraryScreen({super.key});
@override
ConsumerState<AudioLibraryScreen> createState() => _AudioLibraryScreenState();
}
class _AudioLibraryScreenState extends ConsumerState<AudioLibraryScreen>
with SingleTickerProviderStateMixin {
late final TabController _tabController;
// Library tab state
final _searchController = TextEditingController();
List<Map<String, dynamic>> _tracks = [];
bool _loading = false;
bool _unavailable = false;
String? _previewingId;
VideoPlayerController? _previewController;
@override
void initState() {
super.initState();
_tabController = TabController(length: 2, vsync: this);
_fetchTracks('');
}
@override
void dispose() {
_tabController.dispose();
_searchController.dispose();
_previewController?.dispose();
super.dispose();
}
Future<void> _fetchTracks(String q) async {
setState(() { _loading = true; _unavailable = false; });
try {
final api = ref.read(apiServiceProvider);
final data = await api.callGoApi('/audio/library', method: 'GET', queryParams: {'q': q});
final results = (data['results'] as List?)?.cast<Map<String, dynamic>>() ?? [];
setState(() {
_tracks = results;
// 503 is returned as an empty list with an "error" key
_unavailable = data['error'] != null && results.isEmpty;
});
} catch (_) {
setState(() { _unavailable = true; _tracks = []; });
} finally {
if (mounted) setState(() => _loading = false);
}
}
Future<void> _togglePreview(Map<String, dynamic> track) async {
final id = track['id']?.toString() ?? '';
if (_previewingId == id) {
// Stop preview
await _previewController?.pause();
await _previewController?.dispose();
setState(() { _previewController = null; _previewingId = null; });
return;
}
await _previewController?.dispose();
setState(() { _previewingId = id; _previewController = null; });
// Use the Go proxy listen URL — VideoPlayerController handles it as audio
final listenUrl = '${ApiConfig.baseUrl}/audio/library/$id/listen';
final controller = VideoPlayerController.networkUrl(Uri.parse(listenUrl));
try {
await controller.initialize();
await controller.play();
if (mounted) setState(() => _previewController = controller);
} catch (_) {
await controller.dispose();
if (mounted) {
setState(() { _previewingId = null; });
ScaffoldMessenger.of(context).showSnackBar(
const SnackBar(content: Text('Preview unavailable for this track')),
);
}
}
}
void _useTrack(Map<String, dynamic> track) {
final id = track['id']?.toString() ?? '';
final title = (track['title'] as String?) ?? 'Unknown Track';
final artist = (track['artist']?['name'] as String?) ?? '';
final displayTitle = artist.isNotEmpty ? '$title — $artist' : title;
final listenUrl = '${ApiConfig.baseUrl}/audio/library/$id/listen';
Navigator.of(context).pop(AudioTrack(path: listenUrl, title: displayTitle));
}
Future<void> _pickDeviceAudio() async {
final result = await FilePicker.platform.pickFiles(
type: FileType.audio,
allowMultiple: false,
);
if (result != null && result.files.isNotEmpty && mounted) {
final file = result.files.first;
final path = file.path;
if (path != null) {
Navigator.of(context).pop(AudioTrack(
path: path,
title: file.name,
));
}
}
}
@override
Widget build(BuildContext context) {
return Scaffold(
appBar: AppBar(
title: const Text('Add Music'),
bottom: TabBar(
controller: _tabController,
tabs: const [
Tab(icon: Icon(Icons.smartphone), text: 'Device'),
Tab(icon: Icon(Icons.library_music), text: 'Library'),
],
),
),
body: TabBarView(
controller: _tabController,
children: [
_DeviceTab(onPick: _pickDeviceAudio),
_libraryTab(),
],
),
);
}
Widget _libraryTab() {
return Column(
children: [
Padding(
padding: const EdgeInsets.all(12),
child: TextField(
controller: _searchController,
decoration: InputDecoration(
hintText: 'Search tracks...',
prefixIcon: const Icon(Icons.search),
suffixIcon: IconButton(
icon: const Icon(Icons.clear),
onPressed: () {
_searchController.clear();
_fetchTracks('');
},
),
border: OutlineInputBorder(borderRadius: BorderRadius.circular(12)),
contentPadding: const EdgeInsets.symmetric(vertical: 10, horizontal: 16),
),
textInputAction: TextInputAction.search,
onSubmitted: _fetchTracks,
),
),
Expanded(child: _libraryBody()),
],
);
}
Widget _libraryBody() {
if (_loading) return const Center(child: CircularProgressIndicator());
if (_unavailable) {
return const Center(
child: Padding(
padding: EdgeInsets.all(32),
child: Column(
mainAxisSize: MainAxisSize.min,
children: [
Icon(Icons.cloud_off, size: 48, color: Colors.grey),
SizedBox(height: 16),
Text(
'Music library coming soon',
style: TextStyle(fontSize: 16, fontWeight: FontWeight.w600),
),
SizedBox(height: 8),
Text(
'Use the Device tab to add your own audio, or check back after the library is deployed.',
textAlign: TextAlign.center,
style: TextStyle(color: Colors.grey),
),
],
),
),
);
}
if (_tracks.isEmpty) {
return const Center(child: Text('No tracks found'));
}
return ListView.separated(
itemCount: _tracks.length,
separatorBuilder: (_, __) => const Divider(height: 1),
itemBuilder: (context, i) {
final track = _tracks[i];
final id = track['id']?.toString() ?? '';
final title = (track['title'] as String?) ?? 'Unknown';
final artist = (track['artist']?['name'] as String?) ?? '';
final duration = track['duration'] as int? ?? 0;
final mins = (duration ~/ 60).toString().padLeft(2, '0');
final secs = (duration % 60).toString().padLeft(2, '0');
final isPreviewing = _previewingId == id;
return ListTile(
leading: const Icon(Icons.music_note),
title: Text(title, maxLines: 1, overflow: TextOverflow.ellipsis),
subtitle: Text('$artist • $mins:$secs', style: const TextStyle(fontSize: 12)),
trailing: Row(
mainAxisSize: MainAxisSize.min,
children: [
IconButton(
icon: Icon(isPreviewing ? Icons.stop : Icons.play_arrow),
tooltip: isPreviewing ? 'Stop' : 'Preview',
onPressed: () => _togglePreview(track),
),
TextButton(
onPressed: () => _useTrack(track),
child: const Text('Use'),
),
],
),
);
},
);
}
}
class _DeviceTab extends StatelessWidget {
final VoidCallback onPick;
const _DeviceTab({required this.onPick});
@override
Widget build(BuildContext context) {
return Center(
child: Column(
mainAxisSize: MainAxisSize.min,
children: [
const Icon(Icons.folder_open, size: 64, color: Colors.grey),
const SizedBox(height: 16),
const Text('Pick an audio file from your device',
style: TextStyle(fontSize: 16)),
const SizedBox(height: 8),
const Text('MP3, AAC, WAV, FLAC and more',
style: TextStyle(color: Colors.grey)),
const SizedBox(height: 24),
ElevatedButton.icon(
onPressed: onPick,
icon: const Icon(Icons.audio_file),
label: const Text('Browse Files'),
),
],
),
);
}
}

73
sojorn_app/lib/screens/quips/create/enhanced_quip_recorder_screen.dart
View file

@ -10,6 +10,7 @@ import 'package:sojorn/services/video_stitching_service.dart';
import 'package:video_player/video_player.dart';
import '../../../theme/tokens.dart';
import '../../../theme/app_theme.dart';
import '../../audio/audio_library_screen.dart';
import 'quip_preview_screen.dart';
class EnhancedQuipRecorderScreen extends StatefulWidget {
@ -58,6 +59,10 @@ class _EnhancedQuipRecorderScreenState extends State<EnhancedQuipRecorderScreen>
Color _textColor = Colors.white;
double _textPositionY = 0.8; // 0=top, 1=bottom
// Audio Overlay
AudioTrack? _selectedAudio;
double _audioVolume = 0.5;
// Processing State
bool _isProcessing = false;
@ -265,6 +270,8 @@ class _EnhancedQuipRecorderScreenState extends State<EnhancedQuipRecorderScreen>
'color': _textColor.value.toHex(),
'position': _textPositionY,
} : null,
audioOverlayPath: _selectedAudio?.path,
audioVolume: _audioVolume,
);
if (finalFile != null && mounted) {
@ -354,6 +361,16 @@ class _EnhancedQuipRecorderScreenState extends State<EnhancedQuipRecorderScreen>
});
}
Future<void> _pickAudio() async {
final result = await Navigator.push<AudioTrack>(
context,
MaterialPageRoute(builder: (_) => const AudioLibraryScreen()),
);
if (result != null && mounted) {
setState(() => _selectedAudio = result);
}
}
@override
Widget build(BuildContext context) {
if (_isInitializing) {
@ -569,6 +586,41 @@ class _EnhancedQuipRecorderScreenState extends State<EnhancedQuipRecorderScreen>
],
),
// Audio track chip (shown when audio is selected)
if (_selectedAudio != null && !_isRecording)
Padding(
padding: const EdgeInsets.only(top: 8),
child: Row(
mainAxisAlignment: MainAxisAlignment.center,
children: [
Chip(
backgroundColor: Colors.deepPurple.shade700,
avatar: const Icon(Icons.music_note, color: Colors.white, size: 16),
label: Text(
_selectedAudio!.title,
maxLines: 1,
overflow: TextOverflow.ellipsis,
style: const TextStyle(color: Colors.white, fontSize: 12),
),
deleteIcon: const Icon(Icons.close, color: Colors.white70, size: 16),
onDeleted: () => setState(() => _selectedAudio = null),
),
const SizedBox(width: 8),
SizedBox(
width: 80,
child: Slider(
value: _audioVolume,
min: 0.0,
max: 1.0,
activeColor: Colors.deepPurple.shade300,
inactiveColor: Colors.white24,
onChanged: (v) => setState(() => _audioVolume = v),
),
),
],
),
),
// Additional controls row
if (_recordedSegments.isNotEmpty && !_isRecording)
Row(
@ -590,7 +642,24 @@ class _EnhancedQuipRecorderScreenState extends State<EnhancedQuipRecorderScreen>
),
),
),
// Music / audio overlay button
GestureDetector(
onTap: _pickAudio,
child: Container(
padding: const EdgeInsets.all(8),
decoration: BoxDecoration(
color: _selectedAudio != null ? Colors.deepPurple : Colors.white24,
shape: BoxShape.circle,
),
child: Icon(
Icons.music_note,
color: _selectedAudio != null ? Colors.white : Colors.white70,
size: 20,
),
),
),
// Camera toggle
GestureDetector(
onTap: _toggleCamera,
@ -607,7 +676,7 @@ class _EnhancedQuipRecorderScreenState extends State<EnhancedQuipRecorderScreen>
),
),
),
// Flash toggle
GestureDetector(
onTap: _toggleFlash,

29
sojorn_app/lib/services/video_stitching_service.dart
View file

@ -11,8 +11,10 @@ class VideoStitchingService {
List<Duration> segmentDurations,
String filter,
double playbackSpeed,
Map<String, dynamic>? textOverlay,
) async {
Map<String, dynamic>? textOverlay, {
String? audioOverlayPath,
double audioVolume = 0.5,
}) async {
if (segments.isEmpty) return null;
if (segments.length == 1 && filter == 'none' && playbackSpeed == 1.0 && textOverlay == null) {
return segments.first;
@ -105,13 +107,30 @@ class VideoStitchingService {
final session = await FFmpegKit.execute(command);
final returnCode = await session.getReturnCode();
if (ReturnCode.isSuccess(returnCode)) {
return outputFile;
} else {
if (!ReturnCode.isSuccess(returnCode)) {
final logs = await session.getOutput();
print('FFmpeg error: $logs');
return null;
}
// Audio overlay pass (optional second FFmpeg call to mix in background audio)
if (audioOverlayPath != null && audioOverlayPath.isNotEmpty) {
final audioOutputFile = File('${tempDir.path}/audio_${DateTime.now().millisecondsSinceEpoch}.mp4');
final vol = audioVolume.clamp(0.0, 1.0).toStringAsFixed(2);
final audioCmd =
"-i '${outputFile.path}' -i '$audioOverlayPath' "
"-filter_complex '[1:a]volume=${vol}[a1];[0:a][a1]amix=inputs=2:duration=first:dropout_transition=0' "
"-c:v copy -shortest '${audioOutputFile.path}'";
final audioSession = await FFmpegKit.execute(audioCmd);
final audioCode = await audioSession.getReturnCode();
if (ReturnCode.isSuccess(audioCode)) {
return audioOutputFile;
}
// If audio mix fails, fall through and return the video without the overlay
print('Audio overlay mix failed — returning video without audio overlay');
}
return outputFile;
} catch (e) {
print('Video stitching error: $e');
return null;